Skip to content

Commit 0de6abc

Browse files
committed
Remove contrived primary topic from topics rake task
We've updated the evaluation repo topic tagger to no longer require a primary topic. It now handes error statuses and error messages appropriately. This means we can remove this temporary solution.
1 parent 6d53c57 commit 0de6abc

2 files changed

Lines changed: 0 additions & 41 deletions

File tree

lib/tasks/evaluation.rake

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -105,16 +105,6 @@ namespace :evaluation do
105105
raise "Requires an INPUT env var" if ENV["INPUT"].blank?
106106

107107
result = AutoEvaluation::TopicTagger.call(ENV["INPUT"])
108-
109-
# This is a temporary solution to ensure that the evaluation repo topic tagger evaluations continue to work
110-
# since primary_topic is a required field on the GenerateInput class. This will be addressed in a follow up bit
111-
# of work.
112-
if result.status == "error"
113-
result = result.to_h.merge(
114-
primary_topic: "invalid_tool_output",
115-
)
116-
end
117-
118108
puts(result.to_json)
119109
end
120110

spec/lib/tasks/evaluation_spec.rb

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -431,37 +431,6 @@
431431
.to output("#{result.to_json}\n").to_stdout
432432
end
433433
end
434-
435-
context "when the result has an error status" do
436-
it "sets the primary_topic to 'invalid_tool_output'" do
437-
ClimateControl.modify(INPUT: input) do
438-
allow(AutoEvaluation::TopicTagger).to receive(:call)
439-
.with(input)
440-
.and_return(
441-
AutoEvaluation::TopicTagger::Result.new(
442-
status: "error",
443-
primary_topic: nil,
444-
secondary_topic: nil,
445-
metrics: {},
446-
llm_response: {},
447-
error_message: "LLM did not return valid JSON that conformed to the schema.",
448-
),
449-
)
450-
451-
expected_result = {
452-
status: "error",
453-
primary_topic: "invalid_tool_output",
454-
secondary_topic: nil,
455-
metrics: {},
456-
llm_response: {},
457-
error_message: "LLM did not return valid JSON that conformed to the schema.",
458-
}.to_json
459-
460-
expect { Rake::Task[task_name].invoke }
461-
.to output("#{expected_result}\n").to_stdout
462-
end
463-
end
464-
end
465434
end
466435

467436
describe "batch_process" do

0 commit comments

Comments
 (0)