Implement quota for auto evaluation

This adds the ability to limit the number of autoevaluations
that can be run per hour. If the limit is reached, further evaluations
are skipped and a warning is logged.

We've set the limit to 300 evaluations per hour based on discussion with
data science.

We're treating topic tagging as an auto evaluation. In order to reuse the
quota functionality i've updated the AnswerTopicsJob to ingerit from the
base job and added the quota checks.

Author: davidgisbey

app/jobs/answer_analysis/answer_relevancy_job.rb

@@ -2,6 +2,7 @@ module AnswerAnalysis
   class AnswerRelevancyJob < BaseJob
     def perform(answer_id)
       return unless eligible_for_answer_analysis?(answer_id)
+      return if quota_limit_reached?
 
       answer = Answer.includes(:question, :answer_relevancy_aggregate).find(answer_id)
       return logger.warn(aggregate_exists_warn_message(answer.id)) if answer.answer_relevancy_aggregate.present?

app/jobs/answer_analysis/base_job.rb

@@ -15,5 +15,18 @@ def eligible_for_answer_analysis?(answer_id)
 
       eligible
     end
+
+    def quota_limit_reached?
+      current_count = Rails.cache.read("auto_evaluations_count")
+      quota = Rails.configuration.max_auto_evaluations_per_hour
+
+      if current_count.present? && current_count >= quota
+        logger.warn("Auto-evaluation quota limit of #{quota} evaluations per hour reached")
+        return true
+      end
+
+      Rails.cache.increment("auto_evaluations_count", expires_at: Time.current.at_end_of_hour)
+      false
+    end
   end
 end

app/jobs/answer_analysis/tag_topics_job.rb

@@ -1,6 +1,5 @@
 module AnswerAnalysis
-  class TagTopicsJob < ApplicationJob
-    MAX_RETRIES = 5
+  class TagTopicsJob < BaseJob
     retry_on Anthropic::Errors::APIError, wait: 1.minute, attempts: MAX_RETRIES
 
     def perform(answer_id)
@@ -11,6 +10,7 @@ def perform(answer_id)
       unless answer.eligible_for_topic_analysis?
         return logger.info("Answer #{answer_id} is not eligible for topic analysis")
       end
+      return if quota_limit_reached?
 
       result = AutoEvaluation::TopicTagger.call(answer.question_used)
 

config/application.rb

@@ -91,5 +91,7 @@ class Application < Rails::Application
                                              .topic_tagger
                                              .dig("tool_spec", "input_schema", "$defs", "govuk_topic_tags", "enum")
                                              .sort
+
+    config.max_auto_evaluations_per_hour = 300
   end
 end

spec/jobs/answer_analysis/answer_relevancy_job_spec.rb

@@ -17,6 +17,7 @@
   end
 
   it_behaves_like "a job in queue", "default"
+  it_behaves_like "a job that adheres to the auto_evaluation quota", AutoEvaluation::AnswerRelevancy
 
   describe "#perform" do
     it "calls AutoEvaluation::AnswerRelevancy the configured number of times with the correct arguments" do

spec/jobs/answer_analysis/tag_topics_job_spec.rb

@@ -19,6 +19,7 @@
   before { allow(AutoEvaluation::TopicTagger).to receive(:call).and_return(topic_tagger_result) }
 
   it_behaves_like "a job in queue", "default"
+  it_behaves_like "a job that adheres to the auto_evaluation quota", AutoEvaluation::TopicTagger
 
   describe "#perform" do
     it "calls the AutoEvaluation::TopicTagger with the answer message" do

spec/support/job_examples.rb

@@ -4,4 +4,42 @@ module JobExamples
       expect(described_class.queue_name).to eq(expected_queue)
     end
   end
+
+  shared_examples "a job that adheres to the auto_evaluation quota" do |metric|
+    let(:answer) { create(:answer) }
+
+    before do
+      memory_store = ActiveSupport::Cache::MemoryStore.new
+      allow(Rails).to receive(:cache).and_return(memory_store)
+    end
+
+    it "writes the auto_evaluations_count cache key on the first evaluation" do
+      described_class.new.perform(answer.id)
+      expect(Rails.cache.read("auto_evaluations_count")).to eq(1)
+    end
+
+    it "increments the auto_evaluations_count cache key for subsequent evaluations" do
+      Rails.cache.write("auto_evaluations_count", 1)
+      described_class.new.perform(answer.id)
+      expect(Rails.cache.read("auto_evaluations_count")).to eq(2)
+    end
+
+    it "logs a warning and does not perform the evaluation when quota limit is reached" do
+      max_evaluations = Rails.configuration.max_auto_evaluations_per_hour
+      Rails.cache.write("auto_evaluations_count", max_evaluations)
+      expect(described_class.logger)
+        .to receive(:warn)
+        .with("Auto-evaluation quota limit of #{max_evaluations} evaluations per hour reached")
+      expect(metric).not_to receive(:call)
+
+      described_class.new.perform(answer.id)
+    end
+
+    it "expires the auto_evaluations_count cache key at the end of the hour" do
+      described_class.new.perform(answer.id)
+      travel_to(Time.current.at_end_of_hour + 1.minute) do
+        expect(Rails.cache.read("auto_evaluations_count")).to be_nil
+      end
+    end
+  end
 end