Integraate Answer Relevancy Analysis into analysis workflow

This updates the compose answer job to call the answer relevancy job
after an answer has been successfully composed and persisted.

Author: davidgisbey

app/jobs/compose_answer_job.rb

@@ -14,6 +14,11 @@ def perform(question_id)
       logger.warn("Already an answer created for #{question_id}")
     end
 
-    AnswerAnalysis::TagTopicsJob.perform_later(answer.id) if answer.persisted?
+    if answer.persisted?
+      # TODO: Once we've added a few metrics we should move these to a single job that
+      # kicks off all analysis jobs.
+      AnswerAnalysis::TagTopicsJob.perform_later(answer.id)
+      AnswerAnalysis::AnswerRelevancyJob.perform_later(answer.id)
+    end
   end
 end

spec/jobs/compose_answer_job_spec.rb

@@ -6,6 +6,7 @@
   before do
     allow(AnswerComposition::Composer).to receive(:call).and_return(returned_answer)
     allow(AnswerAnalysis::TagTopicsJob).to receive(:perform_later)
+    allow(AnswerAnalysis::AnswerRelevancyJob).to receive(:perform_later)
   end
 
   it_behaves_like "a job in queue", "answer"
@@ -22,6 +23,11 @@
       expect(AnswerAnalysis::TagTopicsJob).to have_received(:perform_later).with(returned_answer.id)
     end
 
+    it "calls the AnswerAnalysis::AnswerRelevancyJob with the answer_id" do
+      described_class.new.perform(question.id)
+      expect(AnswerAnalysis::AnswerRelevancyJob).to have_received(:perform_later).with(returned_answer.id)
+    end
+
     context "when the question has already been answered" do
       let(:question) { create(:question, :with_answer) }
 

spec/requests/api/v1/conversation_flow_spec.rb

@@ -74,6 +74,7 @@ def when_i_create_a_conversation
       )
     end
     allow(AnswerAnalysis::TagTopicsJob).to receive(:perform_later)
+    allow(AnswerAnalysis::AnswerRelevancyJob).to receive(:perform_later)
 
     post api_v1_create_conversation_path,
          params: { user_question: "What is the capital of France?" },

spec/system/conversation_js_features_spec.rb

@@ -282,6 +282,10 @@ def stubs_for_mock_answer(question,
 
     stub_claude_output_guardrails(answer)
     stub_claude_messages_topic_tagger(question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: question,
+      answer_message: answer,
+    )
   end
 
   def then_i_cant_see_the_clear_chat_link

spec/system/conversation_with_claude_structured_answer_spec.rb

@@ -51,6 +51,10 @@ def when_the_first_answer_is_generated
     stub_claude_structured_answer(@first_question, @first_answer)
     stub_claude_output_guardrails(@first_answer, "False | None")
     stub_claude_messages_topic_tagger(@first_question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: @first_question,
+      answer_message: @first_answer,
+    )
 
     execute_queued_sidekiq_jobs
   end
@@ -83,6 +87,10 @@ def when_the_second_answer_is_generated
     stub_claude_structured_answer(rephrased_question, @second_answer)
     stub_claude_output_guardrails(@second_answer, "False | None")
     stub_claude_messages_topic_tagger(rephrased_question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: rephrased_question,
+      answer_message: @second_answer,
+    )
 
     execute_queued_sidekiq_jobs
   end

spec/system/conversation_with_open_ai_with_structured_answer_spec.rb

@@ -55,6 +55,10 @@ def when_the_first_answer_is_generated
     )
     stub_openai_output_guardrail("Lots of tax.")
     stub_claude_messages_topic_tagger(@first_question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: @first_question,
+      answer_message: "Lots of tax.",
+    )
 
     execute_queued_sidekiq_jobs
   end
@@ -75,6 +79,10 @@ def when_the_second_answer_is_generated
     )
     stub_openai_output_guardrail("Even more tax.")
     stub_claude_messages_topic_tagger(rephrased_question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: rephrased_question,
+      answer_message: "Even more tax.",
+    )
 
     execute_queued_sidekiq_jobs
   end

spec/system/user_conversation_activity_is_shown_in_admin_spec.rb

@@ -52,6 +52,10 @@ def and_the_answer_is_generated
     stub_claude_structured_answer(@question, @answer)
     stub_claude_output_guardrails(@answer, "False | None")
     stub_claude_messages_topic_tagger(@question)
+    stub_bedrock_invoke_model_openai_oss_answer_relevancy(
+      question_message: @question,
+      answer_message: @answer,
+    )
 
     execute_queued_sidekiq_jobs
   end