Merge pull request #1105 from alphagov/538-retry-cleanup-work

[CHAT-538] Pass less information to logs for BedrockOpenAIOssInvoke retries

Author: davidgisbey

lib/auto_evaluation/bedrock_openai_oss_invoke.rb

@@ -35,7 +35,6 @@ def call
       client = Aws::BedrockRuntime::Client.new
 
       attempts = 0
-      last_error = nil
 
       while attempts <= MAX_ATTEMPTS
         attempts += 1
@@ -70,23 +69,17 @@ def call
             metrics: build_metrics(start_time, parsed_response),
           )
         rescue *RESCUABLE_ERRORS => e
-          error_string = "#{e.class}, #{e.message}"
           full_error_message = "LLM did not return valid JSON that conformed to the schema. " \
-                          "Attempt #{attempts}/#{MAX_ATTEMPTS}. Error: #{error_string}."
-
-          if last_error.present? && error_string != last_error
-            full_error_message += " This error is different from the previous error: #{last_error}."
-          end
+                               "Attempt #{attempts}/#{MAX_ATTEMPTS}."
 
           logger.warn(full_error_message)
 
           if attempts >= MAX_ATTEMPTS
             raise InvalidLlmResponseError, "LLM did not return valid JSON that conformed to the schema " \
-                                           "after #{MAX_ATTEMPTS} attempts. Error: #{error_string}"
+                                           "after #{MAX_ATTEMPTS} attempts. Error: #{e.class}, #{e.message}"
 
           end
 
-          last_error = error_string
           next
         end
       end

spec/lib/auto_evaluation/bedrock_openai_oss_invoke_spec.rb

@@ -44,8 +44,7 @@
         allow(Rails).to receive(:logger).and_return(logger)
         (1..described_class::MAX_ATTEMPTS).each do |i|
           expected_log_message = "LLM did not return valid JSON that conformed to the schema. " \
-                                 "Attempt #{i}/#{described_class::MAX_ATTEMPTS}. " \
-                                 "Error: #{error_class}, #{error_message}"
+                                 "Attempt #{i}/#{described_class::MAX_ATTEMPTS}."
           expect(logger).to receive(:warn)
             .with(/#{expected_log_message}/)
             .ordered
@@ -55,48 +54,6 @@
           described_class.call(user_message:, tool:)
         }.to raise_error(described_class::InvalidLlmResponseError)
       end
-
-      it "logs additional information when the error message (or class) changes between retries" do
-        logger = instance_double(Logger)
-        allow(Rails).to receive(:logger).and_return(logger)
-        allow(logger).to receive(:warn)
-
-        call_count = 0
-        original_response_body = stub.response.body
-
-        stub_request(:post, StubBedrock::OPENAI_GPT_OSS_ENDPOINT_REGEX).to_return do |_request|
-          call_count += 1
-          if call_count == 1
-            { status: 200, headers: { "Content-Type" => "application/json" }, body: original_response_body }
-          else
-            {
-              status: 200,
-              headers: { "Content-Type" => "application/json" },
-              body: {
-                choices: [{
-                  message: {
-                    tool_calls: [{ function: { arguments: "not_json" } }],
-                  },
-                  finish_reason: "stop",
-                }],
-                usage: { prompt_tokens: 10, completion_tokens: 10 },
-              }.to_json,
-            }
-          end
-        end
-
-        error = "#{error_class}, #{error_message}"
-        expect(logger).to receive(:warn)
-          .with(/#{error}/)
-          .ordered
-        expect(logger).to receive(:warn)
-          .with(/This error is different from the previous error: #{error}/)
-          .ordered
-
-        expect {
-          described_class.call(user_message: user_message, tool: tool)
-        }.to raise_error(described_class::InvalidLlmResponseError)
-      end
     end
 
     it "returns a Result object with the evaluation data" do