Merge pull request #949 from alphagov/chat-115

Author: jackbot

lib/answer_composition/pipeline/claude/structured_answer_composer.rb

@@ -36,14 +36,20 @@ def call
 
         context.answer.assign_llm_response("structured_answer", llm_response_with_link_token_mapping)
         tool_output = response[:content][0][:input]
+        answer_completeness = tool_output[:answer_completeness]
+        answered = %w[complete partial].include?(answer_completeness.downcase)
 
-        return abort_cannot_answer(start_time, response) unless tool_output[:answered]
+        unless answered
+          return abort_cannot_answer(start_time, response, answer_completeness)
+        end
 
         set_context_sources(tool_output[:sources_used])
-        return abort_cannot_answer(start_time, response) if context.answer.sources.none?(&:used)
+        if context.answer.sources.none?(&:used)
+          return abort_cannot_answer(start_time, response, answer_completeness)
+        end
 
         message = link_token_mapper.replace_tokens_with_links(tool_output[:answer])
-        context.answer.assign_attributes(message:, status: "answered", completeness: tool_output[:answer_completeness])
+        context.answer.assign_attributes(message:, status: "answered", completeness: answer_completeness)
         context.answer.assign_metrics("structured_answer", build_metrics(start_time, response))
       end
 
@@ -119,10 +125,11 @@ def tools
         [prompt_config[:tool_spec]]
       end
 
-      def abort_cannot_answer(start_time, response)
+      def abort_cannot_answer(start_time, response, answer_completeness)
         context.abort_pipeline!(
           message: Answer::CannedResponses::LLM_CANNOT_ANSWER_MESSAGE,
           status: "unanswerable_llm_cannot_answer",
+          completeness: answer_completeness,
           metrics: { "structured_answer" => build_metrics(start_time, response) },
         )
       end

spec/lib/answer_composition/pipeline/claude/structured_answer_composer_spec.rb

@@ -24,7 +24,7 @@
     before { context.search_results = [search_result] }
 
     shared_examples "llm cannot answer the question" do |options|
-      it "aborts the pipeline and sets the answer's status and message correctly" do
+      it "aborts the pipeline and sets the answer's attributes correctly" do
         stub_claude_structured_answer(
           question.message,
           "Sorry I cannot answer that question.",
@@ -34,6 +34,7 @@
         expect { described_class.call(context) }.to throw_symbol(:abort)
           .and change { context.answer.status }.to("unanswerable_llm_cannot_answer")
           .and change { context.answer.message }.to(Answer::CannedResponses::LLM_CANNOT_ANSWER_MESSAGE)
+          .and change { context.answer.completeness }.to(options[:answer_completeness])
       end
 
       it "sets sources used to false for all sources" do
@@ -110,15 +111,19 @@
       end
     end
 
-    it "uses Claude via Anthropic to assign the correct values to the context's answer" do
-      answer = "VAT (Value Added Tax) is a tax applied to most goods and services in the UK."
-      stub_claude_structured_answer(question.message, answer)
+    describe "with valid answer_completeness values" do
+      %w[complete partial].each do |answer_completeness|
+        it "uses Claude via Anthropic to assign the correct values to the context's answer" do
+          answer = "VAT (Value Added Tax) is a tax applied to most goods and services in the UK."
+          stub_claude_structured_answer(question.message, answer, answer_completeness: answer_completeness)
 
-      described_class.call(context)
+          described_class.call(context)
 
-      expect(context.answer.message.squish).to eq(answer)
-      expect(context.answer)
-        .to have_attributes(status: "answered", completeness: "complete")
+          expect(context.answer.message.squish).to eq(answer)
+          expect(context.answer)
+            .to have_attributes(status: "answered", completeness: answer_completeness)
+        end
+      end
     end
 
     it "stores the LLM response" do
@@ -194,11 +199,11 @@
         .and change { context.answer.sources.first.used }.to(false)
     end
 
-    context "when answered is false" do
+    context "when answer_completeness is not a successful value" do
       include_examples "llm cannot answer the question", {
         answered: false,
         sources_used: [],
-        answer_completeness: "incomplete",
+        answer_completeness: "no_information",
       }
     end