Implement quota for auto-evaluation metrics

This adds the ability to limit the number of auto-evaluation metrics
that can be run per hour. If the limit is reached, further evaluations
are skipped and a warning is logged.

We've set the limit to 300 metrics per hour based on discussion with
data science.

Author: davidgisbey

app/jobs/answer_analysis/answer_relevancy_job.rb

@@ -2,6 +2,7 @@ module AnswerAnalysis
   class AnswerRelevancyJob < BaseMetricJob
     def perform(answer_id)
       return unless eligible_for_answer_analysis?(answer_id)
+      return if quota_limit_reached?
 
       answer = Answer.includes(:question, :answer_relevancy_aggregate).find_by(id: answer_id)
       return logger.warn(aggregate_exists_warn_message(answer.id)) if answer.answer_relevancy_aggregate.present?

app/jobs/answer_analysis/base_metric_job.rb

@@ -15,5 +15,23 @@ def eligible_for_answer_analysis?(answer_id)
 
       eligible
     end
+
+    def quota_limit_reached?
+      quota = Rails.configuration.max_auto_evaluation_metrics_per_hour
+      current_count = Rails.cache.read("auto_evaluation_metrics_run_count")
+
+      if current_count.nil?
+        Rails.cache.write("auto_evaluation_metrics_run_count", 1, expires_in: 1.hour)
+        return false
+      end
+
+      if current_count >= quota
+        logger.warn("Auto-evaluation quota limit of #{quota} metrics per hour reached")
+        return true
+      end
+
+      Rails.cache.increment("auto_evaluation_metrics_run_count")
+      false
+    end
   end
 end

config/application.rb

@@ -91,5 +91,7 @@ class Application < Rails::Application
                                              .topic_tagger
                                              .dig("tool_spec", "input_schema", "$defs", "govuk_topic_tags", "enum")
                                              .sort
+
+    config.max_auto_evaluation_metrics_per_hour = 300
   end
 end

spec/jobs/answer_analysis/answer_relevancy_job_spec.rb

@@ -53,6 +53,7 @@
     allow(AutoEvaluation::AnswerRelevancy)
       .to receive(:call).and_return(first_result, second_result, third_result)
     stub_const("AnswerAnalysis::BaseMetricJob::NUMBER_OF_RUNS", 3)
+    allow(Rails.configuration).to receive(:max_auto_evaluation_metrics_per_hour).and_return(2)
   end
 
   it_behaves_like "a job in queue", "default"
@@ -98,6 +99,31 @@
       end
     end
 
+    it "writes the auto_evaluation_metrics_run_count cache key on the first metric run" do
+      expect(Rails.cache).to receive(:write)
+                         .with("auto_evaluation_metrics_run_count", 1, expires_in: 1.hour)
+
+      described_class.new.perform(answer.id)
+    end
+
+    it "increments the auto_evaluation_metrics_run_count cache key in subsequent runs" do
+      allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(1)
+      expect(Rails.cache).to receive(:increment)
+                         .with("auto_evaluation_metrics_run_count")
+
+      described_class.new.perform(answer.id)
+    end
+
+    it "logs info and does not perform evaluation when quota limit is reached" do
+      allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(2)
+      expect(described_class.logger)
+        .to receive(:warn)
+        .with("Auto-evaluation quota limit of 2 metrics per hour reached")
+      expect(AutoEvaluation::AnswerRelevancy).not_to receive(:call)
+
+      described_class.new.perform(answer.id)
+    end
+
     context "when the answer has a rephrased_question" do
       let(:rephrased_question) { "This is a rephrased_question" }