Skip to content

Commit 009adca

Browse files
committed
Extend quota limit to AnswerTopicsJob
We're treating topic tagging as a metric. This means we need to extend the quota to include AnswerTopicsJob so that both jobs implement quota adherence. I've added a shared example to tests related to the quota.
1 parent a888832 commit 009adca

4 files changed

Lines changed: 35 additions & 27 deletions

File tree

app/jobs/answer_analysis/answer_topics_job.rb

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module AnswerAnalysis
2-
class AnswerTopicsJob < ApplicationJob
2+
class AnswerTopicsJob < BaseMetricJob
33
MAX_RETRIES = 5
44
retry_on Anthropic::Errors::APIError, wait: 1.minute, attempts: MAX_RETRIES
55

@@ -11,6 +11,7 @@ def perform(answer_id)
1111
unless answer.eligible_for_topic_analysis?
1212
return logger.info("Answer #{answer_id} is not eligible for topic analysis")
1313
end
14+
return if quota_limit_reached?
1415

1516
result = AutoEvaluation::TopicTagger.call(answer.rephrased_question || answer.question.message)
1617
topics = answer.build_topics(

spec/jobs/answer_analysis/answer_relevancy_job_spec.rb

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@
5353
allow(AutoEvaluation::AnswerRelevancy)
5454
.to receive(:call).and_return(first_result, second_result, third_result)
5555
stub_const("AnswerAnalysis::BaseMetricJob::NUMBER_OF_RUNS", 3)
56-
allow(Rails.configuration).to receive(:max_auto_evaluation_metrics_per_hour).and_return(2)
5756
end
5857

5958
it_behaves_like "a job in queue", "default"
59+
it_behaves_like "a job that adheres to the metric quota", AutoEvaluation::AnswerRelevancy
6060

6161
describe "#perform" do
6262
it "calls AutoEvaluation::AnswerRelevancy the configured number of times with the correct arguments" do
@@ -99,31 +99,6 @@
9999
end
100100
end
101101

102-
it "writes the auto_evaluation_metrics_run_count cache key on the first metric run" do
103-
expect(Rails.cache).to receive(:write)
104-
.with("auto_evaluation_metrics_run_count", 1, expires_in: 1.hour)
105-
106-
described_class.new.perform(answer.id)
107-
end
108-
109-
it "increments the auto_evaluation_metrics_run_count cache key in subsequent runs" do
110-
allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(1)
111-
expect(Rails.cache).to receive(:increment)
112-
.with("auto_evaluation_metrics_run_count")
113-
114-
described_class.new.perform(answer.id)
115-
end
116-
117-
it "logs info and does not perform evaluation when quota limit is reached" do
118-
allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(2)
119-
expect(described_class.logger)
120-
.to receive(:warn)
121-
.with("Auto-evaluation quota limit of 2 metrics per hour reached")
122-
expect(AutoEvaluation::AnswerRelevancy).not_to receive(:call)
123-
124-
described_class.new.perform(answer.id)
125-
end
126-
127102
context "when the answer has a rephrased_question" do
128103
let(:rephrased_question) { "This is a rephrased_question" }
129104

spec/jobs/answer_analysis/answer_topics_job_spec.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
before { allow(AutoEvaluation::TopicTagger).to receive(:call).and_return(topic_tagger_result) }
2020

2121
it_behaves_like "a job in queue", "default"
22+
it_behaves_like "a job that adheres to the metric quota", TopicTagger
2223

2324
describe "#perform" do
2425
it "calls the AutoEvaluation::TopicTagger with the answer message" do

spec/support/job_examples.rb

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,35 @@ module JobExamples
44
expect(described_class.queue_name).to eq(expected_queue)
55
end
66
end
7+
8+
shared_examples "a job that adheres to the metric quota" do |metric|
9+
let(:answer) { create(:answer) }
10+
11+
before { allow(Rails.configuration).to receive(:max_auto_evaluation_metrics_per_hour).and_return(2) }
12+
13+
it "writes the auto_evaluation_metrics_run_count cache key on the first metric run" do
14+
expect(Rails.cache).to receive(:write)
15+
.with("auto_evaluation_metrics_run_count", 1, expires_in: 1.hour)
16+
17+
described_class.new.perform(answer.id)
18+
end
19+
20+
it "increments the auto_evaluation_metrics_run_count cache key in subsequent runs" do
21+
allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(1)
22+
expect(Rails.cache).to receive(:increment)
23+
.with("auto_evaluation_metrics_run_count")
24+
25+
described_class.new.perform(answer.id)
26+
end
27+
28+
it "logs info and does not perform evaluation when quota limit is reached" do
29+
allow(Rails.cache).to receive(:read).with("auto_evaluation_metrics_run_count").and_return(2)
30+
expect(described_class.logger)
31+
.to receive(:warn)
32+
.with("Auto-evaluation quota limit of 2 metrics per hour reached")
33+
expect(metric).not_to receive(:call)
34+
35+
described_class.new.perform(answer.id)
36+
end
37+
end
738
end

0 commit comments

Comments
 (0)