Return pass/fail status for each output guardrail

Output guardrail results now include a hash keyed by guardrail name with
a boolean pass/fail flag, rather than an array that only contained the
names of guardrails that failed.

This is in order to provide the exact number of guardrail definitions,
as well as their names to the evaluation tool (1). Without providing
these details they must be hardcoded in the evaluation codebase,
resulting in potential discrepancies if guardrail definitions are
updated.

1: alphagov/govuk-chat-evaluation#24

Author: chaecramb

lib/answer_composition/pipeline/answer_guardrails.rb

@@ -9,7 +9,7 @@ def call(context)
           context.abort_pipeline!(
             message: Answer::CannedResponses::ANSWER_GUARDRAILS_FAILED_MESSAGE,
             status: "guardrails_answer",
-            answer_guardrails_failures: response.guardrails,
+            answer_guardrails_failures: response.triggered_guardrails,
             answer_guardrails_status: :fail,
             metrics: { guardrail_name => build_metrics(start_time, response) },
           )

lib/answer_composition/pipeline/question_routing_guardrails.rb

@@ -11,7 +11,7 @@ def call(context)
           context.answer.assign_attributes(
             message: Answer::CannedResponses::QUESTION_ROUTING_GUARDRAILS_FAILED_MESSAGE,
             status: "guardrails_question_routing",
-            question_routing_guardrails_failures: response.guardrails,
+            question_routing_guardrails_failures: response.triggered_guardrails,
           )
         end
 

lib/guardrails/multiple_checker.rb

@@ -1,7 +1,14 @@
 module Guardrails
   class MultipleChecker
     Result = Data.define(:triggered, :guardrails, :llm_response, :llm_guardrail_result,
-                         :llm_prompt_tokens, :llm_completion_tokens, :llm_cached_tokens)
+                         :llm_prompt_tokens, :llm_completion_tokens, :llm_cached_tokens) do
+      def triggered_guardrails
+        return [] unless guardrails
+
+        guardrails.select { |_, v| v }.keys
+      end
+    end
+
     class ResponseError < StandardError
       attr_reader :llm_response, :llm_prompt_tokens, :llm_completion_tokens, :llm_cached_tokens
 
@@ -94,7 +101,7 @@ def parse_response(llm_response:, llm_guardrail_result:, llm_prompt_tokens:, llm
 
       parts = llm_guardrail_result.split(" | ")
       triggered = parts.first.chomp == "True"
-      guardrails = triggered ? extract_guardrails(parts.second) : []
+      guardrails = to_guardrail_hash(parts.second)
 
       Result.new(
         llm_response: llm_response,
@@ -122,9 +129,12 @@ def response_pattern
       end
     end
 
-    def extract_guardrails(parts)
-      guardrail_numbers = parts.scan(/\d+/).map(&:to_i)
-      prompt.guardrails.select { |guardrail| guardrail.key.in?(guardrail_numbers) }.map(&:name)
+    def to_guardrail_hash(parts)
+      triggered_guardrail_numbers = parts.scan(/\d+/).map(&:to_i)
+
+      prompt.guardrails.each_with_object({}) do |guardrail, guardrails_hash|
+        guardrails_hash[guardrail.name.to_sym] = triggered_guardrail_numbers.include?(guardrail.key)
+      end
     end
   end
 end

spec/factories/output_guardrail_result_factory.rb

@@ -18,13 +18,13 @@
 
     trait :pass do
       triggered { false }
-      guardrails { [] }
+      guardrails { { political: false, appropriate_language: false } }
       llm_guardrail_result { "False | None" }
     end
 
     trait :fail do
       triggered { true }
-      guardrails { %w[political] }
+      guardrails { { political: true, appropriate_language: false } }
       llm_guardrail_result { 'True | "3"' }
     end
   end

spec/lib/guardrails/multiple_checker_spec.rb

@@ -8,18 +8,19 @@
     let(:guardrail_response_hash) do
       {
         llm_response: {
-          "message" => {
-            "role" => "assistant",
-            "content" => "False | None",
+          message: {
+            role: "assistant",
+            content: "False | None",
           },
-          "finish_reason" => "stop",
+          finish_reason: "stop",
         },
         llm_guardrail_result: "False | None",
         llm_prompt_tokens: 13,
         llm_completion_tokens: 7,
         llm_cached_tokens: 10,
       }
     end
+    let(:guardrail_result) { build(:guardrails_multiple_checker_result, :pass) }
 
     it "raises an error if the llm_provider is unknown" do
       expect { described_class.call(input, llm_prompt_name, :unknown_provider) }
@@ -33,11 +34,10 @@
         guardrails_config = {
           system_prompt: "{guardrails} {date}",
           user_prompt: "{input}",
-          guardrails: %w[costs personal unique_answer_guardrail],
+          guardrails: %w[political appropriate_language],
           guardrail_definitions: {
-            "costs" => "This is a costs guardrail",
-            "personal" => "This is a personal guardrail",
-            "unique_answer_guardrail" => "This is a unique answer guardrail",
+            "political" => "This is a political guardrail",
+            "appropriate_language" => "This is an appropriate language guardrail",
           },
         }.with_indifferent_access
 
@@ -49,6 +49,11 @@
         described_class.call(input, llm_prompt_name, llm_provider)
         expect(Guardrails::OpenAI::MultipleChecker).to have_received(:call).with(input, instance_of(Guardrails::MultipleChecker::Prompt))
       end
+
+      it "returns the guardrail result" do
+        result = described_class.call(input, llm_prompt_name, llm_provider)
+        expect(result).to eq(guardrail_result)
+      end
     end
 
     context "when the llm_provider is :claude" do
@@ -58,11 +63,10 @@
         guardrails_config = {
           system_prompt: "{guardrails} {date}",
           user_prompt: "{input}",
-          guardrails: %w[costs personal unique_answer_guardrail],
+          guardrails: %w[political appropriate_language],
           guardrail_definitions: {
-            "costs" => "This is a costs guardrail",
-            "personal" => "This is a personal guardrail",
-            "unique_answer_guardrail" => "This is a unique answer guardrail",
+            "political" => "This is a political guardrail",
+            "appropriate_language" => "This is an appropriate language guardrail",
           },
         }.with_indifferent_access
 
@@ -75,6 +79,11 @@
         expect(Guardrails::Claude::MultipleChecker).to have_received(:call).with(input, instance_of(Guardrails::MultipleChecker::Prompt))
       end
 
+      it "returns the guardrail result" do
+        result = described_class.call(input, llm_prompt_name, llm_provider)
+        expect(result).to eq(guardrail_result)
+      end
+
       context "when the response format is incorrect" do
         it "throws a ResponseError" do
           guardrail_result = 'False | "1, 2"'