Merge pull request #749 from alphagov/update-metrics-to-take-answer-model

Update metrics to take answer model as argument

Author: davidgisbey

lib/auto_evaluation/answer_relevancy.rb

@@ -3,15 +3,14 @@ class AutoEvaluation::AnswerRelevancy
 
   def self.call(...) = new(...).call
 
-  def initialize(question_message:, answer_message:)
-    @question_message = question_message
-    @answer_message = answer_message
+  def initialize(answer)
+    @answer = answer
     @llm_responses = {}
     @metrics = {}
   end
 
   def call
-    statements, llm_responses[:statements], metrics[:statements] = StatementGenerator.call(answer_message:)
+    statements, llm_responses[:statements], metrics[:statements] = StatementGenerator.call(answer_message: answer.message)
 
     if statements.empty?
       return build_maximum_score_result(
@@ -58,9 +57,13 @@ def call
 
 private
 
-  attr_reader :question_message, :answer_message
+  attr_reader :answer
   attr_accessor :llm_responses, :metrics
 
+  def question_message
+    answer.rephrased_question || answer.question.message
+  end
+
   def calculate_score(verdicts)
     verdict_count = verdicts.count
     return 1.0 if verdict_count.zero?

lib/auto_evaluation/coherence.rb

@@ -4,9 +4,8 @@ class Coherence
 
     def self.call(...) = new(...).call
 
-    def initialize(question_message:, answer_message:)
-      @question_message = question_message
-      @answer_message = answer_message
+    def initialize(answer)
+      @answer = answer
     end
 
     def call
@@ -24,7 +23,7 @@ def call
 
   private
 
-    attr_reader :question_message, :answer_message
+    attr_reader :answer
 
     def llm_prompts
       Prompts.config.coherence
@@ -33,7 +32,7 @@ def llm_prompts
     def user_prompt
       sprintf(
         llm_prompts.fetch(:user_prompt),
-        answer: answer_message,
+        answer: answer.message,
         question: question_message,
       )
     end
@@ -48,5 +47,9 @@ def normalise_rubric_score(rubric_score)
 
       (rubric_score.to_d - min_rubric_score) / (max_rubric_score - min_rubric_score)
     end
+
+    def question_message
+      answer.rephrased_question || answer.question.message
+    end
   end
 end

lib/auto_evaluation/evaluate_answer_from_question_message.rb

@@ -22,10 +22,7 @@ def call
         raise TaskFailedError, error_message
       end
 
-      evaluation_class.call(
-        question_message:,
-        answer_message: answer.message,
-      )
+      evaluation_class.call(answer)
     end
 
   private

spec/lib/auto_evaluation/answer_relevancy_spec.rb

@@ -3,6 +3,8 @@
     let(:prompts) { AutoEvaluation::Prompts.config.answer_relevancy }
     let(:question_message) { "This is a test question message." }
     let(:answer_message) { "This is a test answer message." }
+    let(:question) { build(:question, message: question_message) }
+    let(:answer) { build(:answer, question:, message: answer_message) }
 
     let(:statements) { ["This is the first statement.", "This is the second statement."] }
     let(:statements_json) { { statements: }.to_json }
@@ -67,10 +69,7 @@
       allow(Clock).to receive(:monotonic_time)
                   .and_return(200.0, 202.0, 204.0, 206.0, 208.0, 210.0)
 
-      result = described_class.call(
-        question_message:,
-        answer_message:,
-      )
+      result = described_class.call(answer)
 
       expected_llm_responses = {
         statements: JSON.parse(statements_stub.response.body),
@@ -100,6 +99,16 @@
         )
     end
 
+    context "when the answer has a rephrased question" do
+      let(:question_message) { "This is a rephrased test question." }
+      let(:answer) { build(:answer, message: answer_message, rephrased_question: question_message) }
+
+      it "uses the rephrased question in the prompt" do
+        result = described_class.call(answer)
+        expect(result.reason).to eq(reason)
+      end
+    end
+
     context "when 'idk' verdicts are present" do
       let(:verdicts) do
         [
@@ -109,10 +118,7 @@
       end
 
       it "treats 'idk' verdicts as positive in the score" do
-        result = described_class.call(
-          question_message:,
-          answer_message:,
-        )
+        result = described_class.call(answer)
 
         expect(result.score).to eq(0.5)
       end
@@ -124,10 +130,7 @@
       it "returns a result object with the expected attributes" do
         allow(Clock).to receive(:monotonic_time).and_return(200.0, 202.0)
 
-        result = described_class.call(
-          question_message:,
-          answer_message:,
-        )
+        result = described_class.call(answer)
 
         expect(result)
           .to be_a(AutoEvaluation::ScoreResult)
@@ -148,10 +151,7 @@
         allow(Clock).to receive(:monotonic_time)
                     .and_return(200.0, 202.0, 204.0, 206.0)
 
-        result = described_class.call(
-          question_message:,
-          answer_message:,
-        )
+        result = described_class.call(answer)
 
         expect(result)
           .to be_a(AutoEvaluation::ScoreResult)
@@ -177,10 +177,7 @@
       it "returns a result object with the expected attributes" do
         allow(Clock).to receive(:monotonic_time).and_return(200.0, 202.0, 204.0, 206.0)
 
-        result = described_class.call(
-          question_message:,
-          answer_message:,
-        )
+        result = described_class.call(answer)
 
         expect(result)
           .to be_a(AutoEvaluation::ScoreResult)

spec/lib/auto_evaluation/coherence_spec.rb

@@ -1,6 +1,8 @@
 RSpec.describe AutoEvaluation::Coherence, :aws_credentials_stubbed do
   describe ".call" do
     let(:prompts) { AutoEvaluation::Prompts.config.coherence }
+    let(:question) { build(:question, message: question_message) }
+    let(:answer) { build(:answer,  question:, message: answer_message) }
     let(:question_message) { "This is a test question message." }
     let(:answer_message) { "This is a test answer message." }
     let(:reason) { "This is the reason for the score." }
@@ -22,10 +24,7 @@
         response_json,
       )
 
-      result = described_class.call(
-        question_message:,
-        answer_message:,
-      )
+      result = described_class.call(answer)
 
       expected_metrics = {
         coherence: {
@@ -62,14 +61,28 @@
           response_json,
         )
 
-        result = described_class.call(
-          question_message:,
-          answer_message:,
-        )
+        result = described_class.call(answer)
 
         expect(result.score).to eq(expected_score)
         expect(result.success).to eq(expected_score >= described_class::THRESHOLD)
       end
     end
+
+    context "when the answer has a rephrased question" do
+      let(:question_message) { "This is a rephrased test question." }
+      let(:answer) { build(:answer, message: answer_message, rephrased_question: question_message) }
+
+      it "uses the rephrased question in the prompt" do
+        stub = bedrock_invoke_model_openai_oss_tool_call(
+          user_prompt,
+          tools,
+          response_json,
+        )
+
+        described_class.call(answer)
+
+        expect(stub).to have_been_requested
+      end
+    end
   end
 end

spec/lib/auto_evaluation/evaluate_answer_from_question_message_spec.rb

@@ -27,10 +27,7 @@
       question_message:,
     )
 
-    expect(evaluation_klass).to have_received(:call).with(
-      question_message:,
-      answer_message: answer.message,
-    )
+    expect(evaluation_klass).to have_received(:call).with(answer)
   end
 
   it "returns the AutoEvaluation::ScoreResult generated by the evaluation class" do