Get basic implementation working

Author: davidgisbey

app/jobs/answer_relevancy_job.rb

@@ -0,0 +1,30 @@
+class AnswerRelevancyJob < AutoEvaluationMetricJob
+  METRIC_NAME = "answer_relevancy".freeze
+
+  def perform(answer_id)
+    answer = find_answer(answer_id)
+    return if log_warnings(METRIC_NAME, answer)
+
+    results = []
+    3.times do
+      results << AutoEvaluation::AnswerRelevancy.call(
+        question_message: answer.rephrased_question || answer.question.message,
+        answer_message: answer.message,
+      )
+    end
+
+    save_metric(answer, results, METRIC_NAME)
+  end
+
+private
+
+  attr_reader :answer_id
+
+  def attribute_warn_message
+    "Answer #{answer_id} has already been evaluated for relevancy"
+  end
+
+  def score_attribute
+    :answer_relevancy_score
+  end
+end

app/jobs/auto_evaluation_metric_job.rb

@@ -0,0 +1,54 @@
+class AutoEvaluationMetricJob < ApplicationJob
+  MAX_RETRIES = 5
+  retry_on StandardError, wait: 1.minute, attempts: MAX_RETRIES
+
+private
+
+  def find_answer(answer_id)
+    Answer.includes(:topics, question: :conversation).find_by(id: answer_id)
+  end
+
+  def log_warnings(metric, answer)
+    unless answer
+      logger.warn("No answer found for #{answer.id}")
+      return true
+    end
+
+    if answer.metric_aggregates.exists?(metric_name: metric)
+      logger.warn(attribute_warn_message)
+      return true
+    end
+
+    unless answer.status == "answered"
+      logger.info("Answer #{answer.id} is not eligible for auto evaluation")
+      return true
+    end
+
+    false
+  end
+
+  def save_metric(answer, results, metric)
+    return logger.warn(attribute_warn_message) if answer.metric_aggregates.exists?(metric_name: metric)
+
+    mean_score = results.sum { |r| r.score } / results.size.to_f
+    aggregate_metric = answer.metric_aggregates.build(score: mean_score, metric_name: metric)
+
+    aggregate_metric.save!
+
+    results.each do |result|
+      run = aggregate_metric.runs.build(
+        score: result.score,
+        reason: result.reason,
+      )
+
+      result.llm_responses.stringify_keys.each do |name, llm_response|
+        run.assign_llm_response(name, llm_response)
+      end
+      result.metrics.stringify_keys.each do |name, metrics|
+        run.assign_metrics(name, metrics)
+      end
+
+      aggregate_metric.runs << run
+    end
+  end
+end

app/jobs/compose_answer_job.rb

@@ -14,6 +14,9 @@ def perform(question_id)
       logger.warn("Already an answer created for #{question_id}")
     end
 
-    AnswerTopicsJob.perform_later(answer.id) if answer.persisted?
+    if answer.persisted?
+      AnswerTopicsJob.perform_later(answer.id)
+      AnswerRelevancyJob.perform_later(answer.id)
+    end
   end
 end

app/models/auto_evaluation_metric_run.rb

@@ -1,4 +1,6 @@
 class AutoEvaluationMetricRun < ApplicationRecord
+  include LlmCallsRecordable
+
   belongs_to :metric_aggregate,
              class_name: "AutoEvaluationMetricAggregate",
              foreign_key: :auto_evaluation_metric_aggregate_id