Add quota limit for auto evaluations

app/jobs/answer_analysis/answer_relevancy_job.rb

@@ -2,6 +2,7 @@ module AnswerAnalysis
   class AnswerRelevancyJob < BaseJob
     def perform(answer_id)
       return unless eligible_for_answer_analysis?(answer_id)
+      return if quota_limit_reached?
 
       answer = Answer.includes(:question, :answer_relevancy_aggregate).find(answer_id)
       return logger.warn(aggregate_exists_warn_message(answer.id)) if answer.answer_relevancy_aggregate.present?

app/jobs/answer_analysis/base_job.rb

@@ -15,5 +15,19 @@ def eligible_for_answer_analysis?(answer_id)
 
       eligible
     end
+
+    def quota_limit_reached?
+      key = "auto_evaluations_count_#{Time.current.beginning_of_hour.to_i}"
+      max_evaluations = Rails.configuration.max_auto_evaluations_per_hour
+      # fallback to 1 in scenarios where we have a null cache (test environment) and this returns nil
+      count = Rails.cache.increment(key, expires_in: 1.hour) || 1
+
+      if count > max_evaluations
+        logger.warn("Auto-evaluation quota limit of #{max_evaluations} evaluations per hour reached")
+        return true
+      end
+
+      false
+    end
   end
 end

app/jobs/answer_analysis/tag_topics_job.rb

@@ -1,6 +1,5 @@
 module AnswerAnalysis
-  class TagTopicsJob < ApplicationJob
-    MAX_RETRIES = 5
+  class TagTopicsJob < BaseJob
     retry_on Anthropic::Errors::APIError, wait: 1.minute, attempts: MAX_RETRIES
 
     def perform(answer_id)
@@ -11,6 +10,7 @@ def perform(answer_id)
       unless answer.eligible_for_topic_analysis?
         return logger.info("Answer #{answer_id} is not eligible for topic analysis")
       end
+      return if quota_limit_reached?
 
       result = AutoEvaluation::TopicTagger.call(answer.question_used)
 

config/application.rb

@@ -91,5 +91,7 @@ class Application < Rails::Application
                                              .topic_tagger
                                              .dig("tool_spec", "input_schema", "$defs", "govuk_topic_tags", "enum")
                                              .sort
+
+    config.max_auto_evaluations_per_hour = 300
   end
 end

spec/jobs/answer_analysis/answer_relevancy_job_spec.rb

@@ -14,9 +14,18 @@
   before do
     allow(AutoEvaluation::AnswerRelevancy)
       .to receive(:call).and_return(*results)
+    allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new)
   end
 
   it_behaves_like "a job in queue", "default"
+  it_behaves_like "a job that adheres to the auto_evaluation quota", AutoEvaluation::AnswerRelevancy
+  it_behaves_like "a job that retries on errors", Aws::Errors::ServiceError do
+    before do
+      allow(AutoEvaluation::AnswerRelevancy)
+        .to receive(:call)
+        .and_raise(Aws::Errors::ServiceError.new(nil, "error"))
+    end
+  end
 
   describe "#perform" do
     it "calls AutoEvaluation::AnswerRelevancy the configured number of times with the correct arguments" do
@@ -118,21 +127,6 @@
       end
     end
 
-    context "when the AnswerRelevancy metric raises an Aws::Errors::ServiceError" do
-      it "retries the job the max number of times" do
-        allow(AutoEvaluation::AnswerRelevancy)
-          .to receive(:call)
-          .and_raise(Aws::Errors::ServiceError.new(nil, "error"))
-
-        described_class.perform_later(answer.id)
-
-        assert_performed_jobs described_class::MAX_RETRIES do
-          expect { perform_enqueued_jobs }
-            .to raise_error(Aws::Errors::ServiceError)
-        end
-      end
-    end
-
     context "when the answer is not eligible for auto-evaluation" do
       let(:answer) { create(:answer, status: Answer.statuses.except(:answered).keys.sample) }
 

spec/jobs/answer_analysis/tag_topics_job_spec.rb

@@ -16,9 +16,20 @@
     )
   end
 
-  before { allow(AutoEvaluation::TopicTagger).to receive(:call).and_return(topic_tagger_result) }
+  before do
+    allow(AutoEvaluation::TopicTagger).to receive(:call).and_return(topic_tagger_result)
+    allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new)
+  end
 
   it_behaves_like "a job in queue", "default"
+  it_behaves_like "a job that adheres to the auto_evaluation quota", AutoEvaluation::TopicTagger
+  it_behaves_like "a job that retries on errors", Anthropic::Errors::APIError do
+    before do
+      allow(AutoEvaluation::TopicTagger)
+        .to receive(:call)
+        .and_raise(Anthropic::Errors::APIError.new(url: "url"))
+    end
+  end
 
   describe "#perform" do
     it "calls the AutoEvaluation::TopicTagger with the answer message" do
@@ -68,23 +79,6 @@
       end
     end
 
-    context "when AutoEvaluation::TopicTagger raises an Anthropic::Errors::APIError" do
-      it "retries the job the max number of times" do
-        allow(AutoEvaluation::TopicTagger).to receive(:call)
-          .and_raise(Anthropic::Errors::APIError.new(
-                       url: "url",
-                     ))
-
-        (described_class::MAX_RETRIES - 1).times do
-          described_class.perform_later(answer.id)
-          expect { perform_enqueued_jobs }.not_to raise_error
-        end
-
-        described_class.perform_later(answer.id)
-        expect { perform_enqueued_jobs }.to raise_error(Anthropic::Errors::APIError)
-      end
-    end
-
     context "when the answer is not eligible for topic analysis" do
       let(:answer) { create(:answer, status: Answer::STATUSES_EXCLUDED_FROM_TOPIC_ANALYSIS.sample) }
 

spec/support/job_examples.rb

@@ -4,4 +4,87 @@ module JobExamples
       expect(described_class.queue_name).to eq(expected_queue)
     end
   end
+
+  shared_examples "a job that adheres to the auto_evaluation quota" do |metric|
+    let(:answer) { create(:answer) }
+    let(:beginning_of_current_hour) { Time.current.beginning_of_hour }
+
+    it "writes the auto_evaluations_count cache key on the first evaluation" do
+      freeze_time do
+        described_class.new.perform(answer.id)
+
+        key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+        expect(Rails.cache.read(key)).to eq(1)
+      end
+    end
+
+    it "increments the auto_evaluations_count cache key for subsequent evaluations" do
+      freeze_time do
+        key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+        Rails.cache.write(key, 1)
+        described_class.new.perform(answer.id)
+        expect(Rails.cache.read(key)).to eq(2)
+      end
+    end
+
+    it "logs a warning and does not perform the evaluation when quota limit is reached" do
+      freeze_time do
+        max_evaluations = Rails.configuration.max_auto_evaluations_per_hour
+        key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+        Rails.cache.write(key, max_evaluations)
+        expect(described_class.logger)
+          .to receive(:warn)
+          .with("Auto-evaluation quota limit of #{max_evaluations} evaluations per hour reached")
+        expect(metric).not_to receive(:call)
+
+        described_class.new.perform(answer.id)
+      end
+    end
+
+    it "uses a new cache key after the hour changes" do
+      beginning_of_current_hour = Time.current.beginning_of_hour
+
+      travel_to(beginning_of_current_hour) do
+        described_class.new.perform(answer.id)
+        key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+        expect(Rails.cache.read(key)).to eq(1)
+      end
+
+      travel_to(beginning_of_current_hour + 30.minutes) do
+        answer_in_same_hour = create(:answer)
+        described_class.new.perform(answer_in_same_hour.id)
+        key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+        expect(Rails.cache.read(key)).to eq(2)
+      end
+
+      travel_to(beginning_of_current_hour + 1.hour) do
+        answer_in_next_hour = create(:answer)
+        described_class.new.perform(answer_in_next_hour.id)
+        key = "auto_evaluations_count_#{(beginning_of_current_hour + 1.hour).to_i}"
+        expect(Rails.cache.read(key)).to eq(1)
+      end
+    end
+
+    it "expires the cache key an hour after the last evaluation" do
+      described_class.new.perform(answer.id)
+      key = "auto_evaluations_count_#{beginning_of_current_hour.to_i}"
+      expect(Rails.cache.read(key)).to eq(1)
+
+      travel 1.hour + 1.minute do
+        expect(Rails.cache.read(key)).to be_nil
+      end
+    end
+  end
+
+  shared_examples "a job that retries on errors" do |error_class|
+    let(:answer) { create(:answer) }
+    it "retries the job the max number of times on #{error_class}" do
+      described_class.perform_later(answer.id)
+
+      assert_performed_jobs described_class::MAX_RETRIES do
+        expect { perform_enqueued_jobs }
+          .to raise_error(error_class)
+      end
+    end
+  end
 end