Merge pull request #713 from alphagov/add-metrics-data-models-and-integrate-into-workflow

Add answer relevancy models and integrate into analysis workflow

Author: davidgisbey

app/controllers/admin/questions_controller.rb

@@ -7,7 +7,7 @@ def index
   def show
     question_scope = Question.includes(
       conversation: :signon_user,
-      answer: [{ sources: :chunk }, :feedback, :topics],
+      answer: [{ sources: :chunk }, :feedback, :topics, { answer_relevancy_aggregate: :runs }],
     )
 
     @question = question_scope.find(params[:id])

app/jobs/answer_analysis/answer_relevancy_job.rb

@@ -0,0 +1,24 @@
+module AnswerAnalysis
+  class AnswerRelevancyJob < BaseJob
+    def perform(answer_id)
+      return unless eligible_for_answer_analysis?(answer_id)
+
+      answer = Answer.includes(:question, :answer_relevancy_aggregate).find(answer_id)
+      return logger.warn(aggregate_exists_warn_message(answer.id)) if answer.answer_relevancy_aggregate.present?
+
+      results = NUMBER_OF_RUNS.times.map { AutoEvaluation::AnswerRelevancy.call(answer) }
+
+      begin
+        AnswerAnalysis::AnswerRelevancyAggregate.create_mean_aggregate_and_score_runs(answer, results)
+      rescue ActiveRecord::RecordNotUnique
+        logger.warn(aggregate_exists_warn_message(answer.id))
+      end
+    end
+
+  private
+
+    def aggregate_exists_warn_message(answer_id)
+      "Answer #{answer_id} has already been evaluated for relevancy"
+    end
+  end
+end

app/jobs/answer_analysis/base_job.rb

@@ -0,0 +1,19 @@
+module AnswerAnalysis
+  class BaseJob < ApplicationJob
+    NUMBER_OF_RUNS = 3
+    MAX_RETRIES = 5
+    retry_on Aws::Errors::ServiceError, wait: 1.minute, attempts: MAX_RETRIES
+
+  private
+
+    def eligible_for_answer_analysis?(answer_id)
+      eligible = Answer.status_answered.exists?(id: answer_id)
+
+      unless eligible
+        logger.warn("Couldn't find an answer #{answer_id} that was eligible for auto-evaluation")
+      end
+
+      eligible
+    end
+  end
+end

app/jobs/answer_analysis/tag_topics_job.rb

@@ -12,7 +12,7 @@ def perform(answer_id)
         return logger.info("Answer #{answer_id} is not eligible for topic analysis")
       end
 
-      result = AutoEvaluation::TopicTagger.call(answer.rephrased_question || answer.question.message)
+      result = AutoEvaluation::TopicTagger.call(answer.question_used)
 
       topics = answer.build_topics(
         primary_topic: result.primary_topic,

app/jobs/compose_answer_job.rb

@@ -14,6 +14,11 @@ def perform(question_id)
       logger.warn("Already an answer created for #{question_id}")
     end
 
-    AnswerAnalysis::TagTopicsJob.perform_later(answer.id) if answer.persisted?
+    if answer.persisted?
+      # TODO: Once we've added a few metrics we should move these to a single job that
+      # kicks off all analysis jobs.
+      AnswerAnalysis::TagTopicsJob.perform_later(answer.id)
+      AnswerAnalysis::AnswerRelevancyJob.perform_later(answer.id)
+    end
   end
 end

app/models/answer.rb

@@ -55,6 +55,7 @@ def self.response_for_question_routing_label(label)
   has_many :sources, -> { order(relevancy: :asc) }, class_name: "AnswerSource"
   has_one :feedback, class_name: "AnswerFeedback"
   has_one :topics, class_name: "AnswerAnalysis::Topics"
+  has_one :answer_relevancy_aggregate, class_name: "AnswerAnalysis::AnswerRelevancyAggregate"
 
   enum :status,
        {
@@ -196,4 +197,12 @@ def group_used_answer_sources_by_base_path
       }
     end
   end
+
+  def has_analysis?
+    topics.present? || answer_relevancy_aggregate.present?
+  end
+
+  def question_used
+    rephrased_question || question.message
+  end
 end

app/models/answer_analysis/answer_relevancy_aggregate.rb

@@ -0,0 +1,13 @@
+module AnswerAnalysis
+  class AnswerRelevancyAggregate < ApplicationRecord
+    include AutoEvaluationResultsCreatable
+
+    self.table_name = "answer_analysis_answer_relevancy_aggregates"
+
+    belongs_to :answer
+    has_many :runs,
+             -> { order(:created_at) },
+             class_name: "AnswerAnalysis::AnswerRelevancyRun",
+             foreign_key: :answer_analysis_answer_relevancy_aggregate_id
+  end
+end

app/models/answer_analysis/answer_relevancy_run.rb

@@ -0,0 +1,11 @@
+module AnswerAnalysis
+  class AnswerRelevancyRun < ApplicationRecord
+    include LlmCallsRecordable
+
+    self.table_name = "answer_analysis_answer_relevancy_runs"
+
+    belongs_to :aggregate,
+               class_name: "AnswerAnalysis::AnswerRelevancyAggregate",
+               foreign_key: :answer_analysis_answer_relevancy_aggregate_id
+  end
+end

app/models/concerns/auto_evaluation_results_creatable.rb

@@ -0,0 +1,27 @@
+module AutoEvaluationResultsCreatable
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def create_mean_aggregate_and_score_runs(answer, results)
+      mean_score = results.map { |result| result.score.to_d }.sum / results.size
+      aggregate = new(answer:, mean_score:)
+
+      results.each do |result|
+        run = aggregate.runs.build(
+          aggregate:,
+          score: result.score,
+          reason: result.reason,
+        )
+
+        result.llm_responses.stringify_keys.each do |name, llm_response|
+          run.assign_llm_response(name, llm_response)
+        end
+        result.metrics.stringify_keys.each do |name, metrics|
+          run.assign_metrics(name, metrics)
+        end
+      end
+
+      aggregate.save!
+    end
+  end
+end

app/views/admin/questions/_analysis_tab.html.erb

@@ -1,41 +1,51 @@
-<%= render "govuk_publishing_components/components/summary_list", {
-  items: [
-    {
-      field: "Primary topic",
-      value: topics.primary_topic&.humanize,
-    },
-    {
-      field: "Secondary topic",
-      value: topics.secondary_topic&.humanize,
-    },
-  ],
-} %>
+<% if topics.present? %>
+  <%= render "govuk_publishing_components/components/summary_list", {
+    title: "Topics",
+    heading_size: "l",
+    heading_level: 2,
+    margin_bottom: 4,
+    items: [
+      {
+        field: "Primary topic",
+        value: topics.primary_topic.humanize,
+      },
+      {
+        field: "Secondary topic",
+        value: topics.secondary_topic&.humanize,
+      },
+    ],
+  } %>
 
-<% if topics.llm_responses.present? %>
-  <%= render "govuk_publishing_components/components/details", {
-    title: "LLM responses",
-  } do %>
-    <% topics.llm_responses.each do |namespace, response| %>
-      <h3 class="govuk-heading-m"><%= namespace %></h3>
-      <p class="govuk-body">
-        <%= render("components/code_snippet", content: JSON.pretty_generate(response)) %>
-      </p>
+  <% if topics.llm_responses.present? %>
+    <%= render "govuk_publishing_components/components/details", {
+      title: "LLM responses",
+    } do %>
+      <% topics.llm_responses.each do |namespace, response| %>
+        <h3 class="govuk-heading-m"><%= namespace %></h3>
+        <p class="govuk-body">
+          <%= render("components/code_snippet", content: JSON.pretty_generate(response)) %>
+        </p>
+      <% end %>
     <% end %>
   <% end %>
-<% end %>
 
-<% if topics.metrics.present? %>
-  <%= render "govuk_publishing_components/components/details", {
-    title: "Metrics",
-  } do %>
-    <%= render "govuk_publishing_components/components/summary_list", {
-      items: topics.metrics.map do |metric, value|
-        {
-          field: metric,
-          value: value,
-        }
-      end,
-      borderless: true,
-    } %>
+  <% if topics.metrics.present? %>
+    <%= render "govuk_publishing_components/components/details", {
+      title: "Metrics",
+    } do %>
+      <%= render "govuk_publishing_components/components/summary_list", {
+        items: topics.metrics.map do |metric, value|
+          {
+            field: metric,
+            value: value,
+          }
+        end,
+        borderless: true,
+      } %>
+    <% end %>
   <% end %>
 <% end %>
+
+<% if answer_relevancy_aggregate.present? %>
+  <%= render "generic_aggregate_auto_evaluation", aggregate: answer_relevancy_aggregate, title: "Answer relevancy" %>
+<% end %>