Expose answer relevancy metrics in admin UI

I've added an additional tab for answer relevancy metrics in the admin
interface on the question show page.

My thoughts for this are if we don't split out the metrics into their own
tabs then the page will get incredibly noisy. This makes it easier to
navigate.

Due to this, i've renamed the analysis tab to topics.

Author: davidgisbey

app/controllers/admin/questions_controller.rb

@@ -7,7 +7,7 @@ def index
   def show
     question_scope = Question.includes(
       conversation: :signon_user,
-      answer: [{ sources: :chunk }, :feedback, :topics],
+      answer: [{ sources: :chunk }, :feedback, :topics, { answer_relevancy_aggregate: :runs }],
     )
 
     @question = question_scope.find(params[:id])

app/models/answer.rb

@@ -197,4 +197,8 @@ def group_used_answer_sources_by_base_path
       }
     end
   end
+
+  def has_analysis?
+    topics.present? || answer_relevancy_aggregate.present?
+  end
 end

app/views/admin/questions/_analysis_tab.html.erb

@@ -1,41 +1,51 @@
-<%= render "govuk_publishing_components/components/summary_list", {
-  items: [
-    {
-      field: "Primary topic",
-      value: topics.primary_topic&.humanize,
-    },
-    {
-      field: "Secondary topic",
-      value: topics.secondary_topic&.humanize,
-    },
-  ],
-} %>
+<% if topics.present? %>
+  <%= render "govuk_publishing_components/components/summary_list", {
+    title: "Topics",
+    heading_size: "l",
+    heading_level: 2,
+    margin_bottom: 4,
+    items: [
+      {
+        field: "Primary topic",
+        value: topics.primary_topic.humanize,
+      },
+      {
+        field: "Secondary topic",
+        value: topics.secondary_topic&.humanize,
+      },
+    ],
+  } %>
 
-<% if topics.llm_responses.present? %>
-  <%= render "govuk_publishing_components/components/details", {
-    title: "LLM responses",
-  } do %>
-    <% topics.llm_responses.each do |namespace, response| %>
-      <h3 class="govuk-heading-m"><%= namespace %></h3>
-      <p class="govuk-body">
-        <%= render("components/code_snippet", content: JSON.pretty_generate(response)) %>
-      </p>
+  <% if topics.llm_responses.present? %>
+    <%= render "govuk_publishing_components/components/details", {
+      title: "LLM responses",
+    } do %>
+      <% topics.llm_responses.each do |namespace, response| %>
+        <h3 class="govuk-heading-m"><%= namespace %></h3>
+        <p class="govuk-body">
+          <%= render("components/code_snippet", content: JSON.pretty_generate(response)) %>
+        </p>
+      <% end %>
     <% end %>
   <% end %>
-<% end %>
 
-<% if topics.metrics.present? %>
-  <%= render "govuk_publishing_components/components/details", {
-    title: "Metrics",
-  } do %>
-    <%= render "govuk_publishing_components/components/summary_list", {
-      items: topics.metrics.map do |metric, value|
-        {
-          field: metric,
-          value: value,
-        }
-      end,
-      borderless: true,
-    } %>
+  <% if topics.metrics.present? %>
+    <%= render "govuk_publishing_components/components/details", {
+      title: "Metrics",
+    } do %>
+      <%= render "govuk_publishing_components/components/summary_list", {
+        items: topics.metrics.map do |metric, value|
+          {
+            field: metric,
+            value: value,
+          }
+        end,
+        borderless: true,
+      } %>
+    <% end %>
   <% end %>
 <% end %>
+
+<% if answer_relevancy_aggregate.present? %>
+  <%= render "auto_evaluation_metric", aggregate: answer_relevancy_aggregate, title: "Answer relevancy" %>
+<% end %>

app/views/admin/questions/_auto_evaluation_metric.html.erb

@@ -0,0 +1,75 @@
+<%
+  items = [
+    {
+      field: "Mean score",
+      value: aggregate.mean_score,
+    },
+  ]
+
+  items += aggregate.runs.flat_map.with_index(1) do |run, index|
+    [
+      { field: "Run #{index} score", value: run.score },
+      { field: "Run #{index} reason", value: run.reason },
+    ]
+  end
+%>
+
+<%= render "govuk_publishing_components/components/summary_list", {
+  title:,
+  heading_level: 2,
+  margin_bottom: 4,
+  heading_size: "l",
+  items: items,
+} %>
+
+<%= render "govuk_publishing_components/components/details", {
+  title: "LLM responses",
+} do %>
+  <% aggregate.runs.each.with_index(1) do |run, index| %>
+    <%= render "govuk_publishing_components/components/heading", {
+      text: "Run #{index}",
+      font_size: "m",
+      heading_level: 2,
+      margin_bottom: 4,
+    } %>
+
+    <% run.llm_responses.each do |namespace, response| %>
+      <%= render "govuk_publishing_components/components/heading", {
+        text: namespace.capitalize,
+        font_size: "s",
+        heading_level: 3,
+      } %>
+
+      <p class="govuk-body">
+        <%= render("components/code_snippet", content: JSON.pretty_generate(response)) %>
+      </p>
+    <% end %>
+  <% end %>
+<% end %>
+
+<%= render "govuk_publishing_components/components/details", {
+  title: "Metrics",
+} do %>
+  <% aggregate.runs.each.with_index(1) do |run, index| %>
+    <%= render "govuk_publishing_components/components/heading", {
+      text: "Run #{index}",
+      font_size: "m",
+      heading_level: 2,
+    } %>
+
+    <% run.metrics.sort.each do |namespace, metrics| %>
+      <%= render "govuk_publishing_components/components/summary_list", {
+        title: namespace.capitalize,
+        items: metrics.map do |metric, value|
+          {
+            field: metric,
+            value: value,
+          }
+        end,
+        borderless: true,
+        heading_size: "s",
+        margin_bottom: 6,
+      } %>
+    <% end %>
+  <% end %>
+<% end %>

app/views/admin/questions/show.html.erb

@@ -39,8 +39,9 @@ content_for(:active_navigation_item, admin_questions_path)
     content: render(
       "analysis_tab",
       topics: @answer.topics,
+      answer_relevancy_aggregate: @answer.answer_relevancy_aggregate,
     ),
-  } if @answer&.topics.present?
+  } if @answer&.has_analysis?
 %>
 
 <div class="govuk-grid-row">

spec/models/answer_spec.rb

@@ -353,4 +353,23 @@
       end
     end
   end
+
+  describe "#has_analysis?" do
+    it "returns true if topics are present" do
+      answer = build(:answer, :with_topics)
+      expect(answer.has_analysis?).to be(true)
+    end
+
+    it "returns true if answer_relevancy_aggregate is present" do
+      answer = build(
+        :answer, answer_relevancy_aggregate: build(:answer_relevancy_aggregate)
+      )
+      expect(answer.has_analysis?).to be(true)
+    end
+
+    it "returns false if no analysis is present" do
+      answer = build(:answer)
+      expect(answer.has_analysis?).to be(false)
+    end
+  end
 end

spec/requests/admin/questions_spec.rb

@@ -284,7 +284,7 @@
         .and have_content('"id": "call_dqGpbb39drQDafLsjDLtnbGD"')
     end
 
-    it "doesn't render the tabs component when there is no analysis" do
+    it "doesn't render the tabs component when there are no topics or auto-eval aggregate data" do
       question = create(:question, :with_answer)
       get admin_show_question_path(question)
 
@@ -353,14 +353,60 @@
           .to have_selector("#details-tab", text: question.message)
       end
 
-      it "renders the topics in the analysis tab" do
+      it "renders the topics in the topics tab" do
         get admin_show_question_path(question)
 
         expect(response.body)
          .to have_selector("#analysis-tab", text: topics.primary_topic.capitalize)
          .and have_selector("#analysis-tab", text: topics.secondary_topic.capitalize)
       end
     end
+
+    context "when answer relevancy aggregate data is present" do
+      let(:run) do
+        create(
+          :answer_relevancy_run,
+          score: 0.85,
+          reason: "The answer is relevant to the question.",
+          llm_responses: {
+            "statements" => { "statements" => ["The answer is relevant."] },
+            "verdicts" => { "verdicts" => [{ "verdict" => "yes" }] },
+          },
+          metrics: {
+            "statements" => { duration: 1.55556 },
+            "verdicts" => { duration: 1.44445 },
+          },
+        )
+      end
+      let!(:aggregate) do
+        create(
+          :answer_relevancy_aggregate,
+          runs: [run],
+        )
+      end
+      let(:question) { aggregate.answer.question }
+
+      it "renders the answer relevancy metrics and LLM responses" do
+        get admin_show_question_path(question)
+
+        expect(response.body.squish)
+          .to have_content("Answer relevancy")
+          .and have_content("Run 1 score")
+          .and have_content("0.85")
+          .and have_content("Run 1 reason")
+          .and have_content("The answer is relevant to the question.")
+
+        expect(response.body.squish)
+          .to have_content("Statements")
+          .and have_content(/duration.*1\.55556/)
+          .and have_content("Verdicts")
+          .and have_content(/duration.*1\.44445/)
+
+        expect(response.body.squish)
+          .to have_content('{ "statements": [ "The answer is relevant." ] }')
+          .and have_content('{ "verdicts": [ { "verdict": "yes" } ] }')
+      end
+    end
   end
 
   def expect_unprocessable_content_with_date_errors

spec/system/user_conversation_activity_is_shown_in_admin_spec.rb

@@ -18,6 +18,7 @@
 
     when_i_click_the_analysis_tab
     then_i_see_the_topics_have_been_tagged
+    and_i_see_the_answer_relevancy_statistics
     and_i_dont_see_the_answer
   end
 
@@ -105,4 +106,12 @@ def then_i_see_the_topics_have_been_tagged
   def and_i_dont_see_the_answer
     expect(page).not_to have_content(@answer)
   end
+
+  def when_i_click_the_answer_relevancy_tab
+    click_link "Answer relevancy"
+  end
+
+  def and_i_see_the_answer_relevancy_statistics
+    expect(page).to have_content(/Mean score.*1.0/)
+  end
 end