Add Rake task for context relevancy auto eval

Copies the behaviour of the existing auto eval metrics and adds a Rake
task which takes in a question and returns the context relevancy metric
for it.

Author: jackbot

lib/tasks/evaluation.rake

@@ -204,4 +204,20 @@ namespace :evaluation do
       abort e.message
     end
   end
+
+  desc "Run context relevancy evaluation for a user input"
+  task generate_context_relevancy_evaluation: :environment do
+    raise "Requires an INPUT env var" if ENV["INPUT"].blank?
+
+    begin
+      result = AutoEvaluation::EvaluateAnswerFromQuestionMessage.call(
+        evaluation_class: AutoEvaluation::ContextRelevancy,
+        question_message: ENV["INPUT"],
+      )
+
+      puts result.to_json
+    rescue AutoEvaluation::EvaluateAnswerFromQuestionMessage::TaskFailedError => e
+      abort e.message
+    end
+  end
 end

spec/lib/tasks/evaluation_spec.rb

@@ -588,4 +588,11 @@
       let(:evaluation_class) { AutoEvaluation::Coherence }
     end
   end
+
+  describe "generate_context_relevancy_evaluation" do
+    it_behaves_like "a task that returns a ScoreResult" do
+      let(:task_name) { "evaluation:generate_context_relevancy_evaluation" }
+      let(:evaluation_class) { AutoEvaluation::ContextRelevancy }
+    end
+  end
 end