Merge pull request #162 from alphagov/question-routing-confidence-score

Add confidence score to question routing eval output

Author: jackbot

lib/tasks/evaluation.rake

@@ -110,6 +110,11 @@ namespace :evaluation do
 
     raise "Error occurred generating answer: #{answer.error_message}" if answer.status =~ /^error/
 
-    puts({ question_routing_label: answer.question_routing_label }.to_json)
+    result = {
+      classification: answer.question_routing_label,
+      confidence_score: answer.question_routing_confidence_score,
+    }
+
+    puts(result.to_json)
   end
 end

spec/lib/tasks/evaluation_spec.rb

@@ -228,10 +228,10 @@
 
     it "outputs the response as JSON to stdout" do
       ClimateControl.modify(INPUT: input) do
-        answer = build(:answer, question_routing_label: "genuine_rag")
+        answer = build(:answer, question_routing_label: "genuine_rag", question_routing_confidence_score: 0.2)
         allow(AnswerComposition::PipelineRunner).to receive(:call).and_return(answer)
         expect { Rake::Task[task_name].invoke("openai") }
-          .to output("{\"question_routing_label\":\"genuine_rag\"}\n").to_stdout
+          .to output("{\"classification\":\"genuine_rag\",\"confidence_score\":0.2}\n").to_stdout
       end
     end