Add AnswerRelevancyAggregate & AnswerRelevancyRun

This adds a migration to the two new tables needed to store answer relevancy
metrics. It also adds the corresponding models and factories.

We will need to record llm multiple llm responses and metrics for each
run so i've included the LlmCallsRecordable module in the AnswerRelevancyRun
model.

Author: davidgisbey

app/models/answer.rb

@@ -55,6 +55,7 @@ def self.response_for_question_routing_label(label)
   has_many :sources, -> { order(relevancy: :asc) }, class_name: "AnswerSource"
   has_one :feedback, class_name: "AnswerFeedback"
   has_one :topics, class_name: "AnswerTopics"
+  has_one :answer_relevancy_aggregate, class_name: "AnswerAnalysis::AnswerRelevancyAggregate"
 
   enum :status,
        {

app/models/answer_analysis/answer_relevancy_aggregate.rb

@@ -0,0 +1,6 @@
+module AnswerAnalysis
+  class AnswerRelevancyAggregate < ApplicationRecord
+    belongs_to :answer
+    has_many :runs, class_name: "AnswerAnalysis::AnswerRelevancyRun"
+  end
+end

app/models/answer_analysis/answer_relevancy_run.rb

@@ -0,0 +1,9 @@
+module AnswerAnalysis
+  class AnswerRelevancyRun < ApplicationRecord
+    include LlmCallsRecordable
+
+    belongs_to :aggregate,
+               class_name: "AnswerAnalysis::AnswerRelevancyAggregate",
+               foreign_key: :answer_relevancy_aggregate_id
+  end
+end

db/migrate/20251216092915_add_answer_relevancy_tables.rb

@@ -0,0 +1,18 @@
+class AddAnswerRelevancyTables < ActiveRecord::Migration[8.0]
+  def change
+    create_table :answer_relevancy_aggregates, id: :uuid do |t|
+      t.decimal :mean_score, null: false
+      t.references :answer, type: :uuid, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true }
+      t.timestamps
+    end
+
+    create_table :answer_relevancy_runs, id: :uuid do |t|
+      t.decimal :score, null: false
+      t.string :reason, null: false
+      t.jsonb :llm_responses
+      t.jsonb :metrics
+      t.references :answer_relevancy_aggregate, type: :uuid, null: false, foreign_key: { on_delete: :cascade }
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2025_12_15_161508) do
+ActiveRecord::Schema[8.0].define(version: 2025_12_16_092915) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "citext"
   enable_extension "pg_catalog.plpgsql"
@@ -33,6 +33,25 @@
     t.index ["created_at"], name: "index_answer_feedback_on_created_at"
   end
 
+  create_table "answer_relevancy_aggregates", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.decimal "mean_score", null: false
+    t.uuid "answer_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["answer_id"], name: "index_answer_relevancy_aggregates_on_answer_id", unique: true
+  end
+
+  create_table "answer_relevancy_runs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.decimal "score", null: false
+    t.string "reason", null: false
+    t.jsonb "llm_responses"
+    t.jsonb "metrics"
+    t.uuid "answer_relevancy_aggregate_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["answer_relevancy_aggregate_id"], name: "index_answer_relevancy_runs_on_answer_relevancy_aggregate_id"
+  end
+
   create_table "answer_source_chunks", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.uuid "content_id", null: false
     t.string "locale", null: false
@@ -173,6 +192,8 @@
   end
 
   add_foreign_key "answer_feedback", "answers", on_delete: :cascade
+  add_foreign_key "answer_relevancy_aggregates", "answers", on_delete: :cascade
+  add_foreign_key "answer_relevancy_runs", "answer_relevancy_aggregates", on_delete: :cascade
   add_foreign_key "answer_sources", "answer_source_chunks", on_delete: :restrict
   add_foreign_key "answer_sources", "answers", on_delete: :cascade
   add_foreign_key "answer_topics", "answers", on_delete: :cascade

spec/factories/answer_relevancy_aggregate_factory.rb

@@ -0,0 +1,6 @@
+FactoryBot.define do
+  factory :answer_relevancy_aggregate, class: "AnswerAnalysis::AnswerRelevancyAggregate" do
+    answer
+    mean_score { 0.5 }
+  end
+end

spec/factories/answer_relevancy_run_factory.rb

@@ -0,0 +1,7 @@
+FactoryBot.define do
+  factory :answer_relevancy_run, class: "AnswerAnalysis::AnswerRelevancyRun" do
+    association :aggregate, factory: :answer_relevancy_aggregate
+    score { 0.5 }
+    reason { "The answer was okay." }
+  end
+end

spec/models/answer_analysis/answer_relevancy_run_spec.rb

@@ -0,0 +1,5 @@
+RSpec.describe AnswerAnalysis::AnswerRelevancyRun do
+  include_examples "llm calls recordable" do
+    let(:model) { build(:answer_relevancy_run) }
+  end
+end