Merge pull request #221 from alphagov/answer-strategy-on-generate-answer

Accept answer_strategy argument for evaluation:generate_answer

Author: kevindew

lib/tasks/evaluation.rake

@@ -36,10 +36,13 @@ namespace :evaluation do
   end
 
   desc "Generate a single answer to a question returned as JSON, for 3rd party evaluation tools"
-  task generate_answer: :environment do
+  task :generate_answer, %i[answer_strategy] => :environment do |_, args|
     raise "requires a QUESTION env var" if ENV["QUESTION"].blank?
 
-    question = Question.new(message: ENV["QUESTION"], conversation: Conversation.new)
+    answer_strategy = args.fetch(:answer_strategy, Rails.configuration.answer_strategy)
+    warn "No answer strategy argument provided, using #{answer_strategy}" unless args[:answer_strategy]
+
+    question = Question.new(message: ENV["QUESTION"], conversation: Conversation.new, answer_strategy:)
     answer = AnswerComposition::Composer.call(question)
     puts({ message: answer.message }.to_json)
   end

spec/lib/tasks/evaluation_spec.rb

@@ -96,17 +96,48 @@
 
     it "outputs the answer as JSON to stdout" do
       answer = build(:answer)
+      question = "What is the current VAT rate?"
+      answer_strategy = "claude_structured_answer"
 
       allow(AnswerComposition::Composer)
         .to receive(:call)
         .with(an_instance_of(Question))
         .and_return(answer)
 
-      ClimateControl.modify(QUESTION: "What is the current VAT rate?") do
+      ClimateControl.modify(QUESTION: question) do
         answer_json = { message: answer.message }.to_json
-        expect { Rake::Task[task_name].invoke }
+        expect { Rake::Task[task_name].invoke(answer_strategy) }
           .to output("#{answer_json}\n").to_stdout
       end
+
+      expect(AnswerComposition::Composer)
+        .to have_received(:call)
+        .with(an_object_having_attributes(message: question,
+                                          conversation: an_instance_of(Conversation),
+                                          answer_strategy: answer_strategy))
+    end
+
+    it "warns when an answer_strategy argument isn't given" do
+      answer = build(:answer)
+      question = "What is the current VAT rate?"
+      default_answer_strategy = Rails.configuration.answer_strategy
+
+      allow(AnswerComposition::Composer)
+        .to receive(:call)
+        .with(an_instance_of(Question))
+        .and_return(answer)
+
+      ClimateControl.modify(QUESTION: question) do
+        expect { Rake::Task[task_name].invoke }
+          .to output.to_stdout
+          .and output("No answer strategy argument provided, using #{default_answer_strategy}\n").to_stderr
+      end
+
+      expect(AnswerComposition::Composer)
+        .to have_received(:call)
+        .with(an_object_having_attributes(message: question,
+                                          conversation: an_instance_of(Conversation),
+                                          answer_strategy: default_answer_strategy))
     end
   end