From d697e4d6fc4eb9a3271e5641f0f54ea0947efea4 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Tue, 1 Apr 2025 17:56:25 +0200 Subject: [PATCH] Store API query attempts in config.json Closes #440 --- cmd/eval-dev-quality/cmd/config.go | 3 +++ cmd/eval-dev-quality/cmd/evaluate.go | 2 ++ cmd/eval-dev-quality/cmd/evaluate_test.go | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/eval-dev-quality/cmd/config.go b/cmd/eval-dev-quality/cmd/config.go index 0aa9e52b..7a4c4a60 100644 --- a/cmd/eval-dev-quality/cmd/config.go +++ b/cmd/eval-dev-quality/cmd/config.go @@ -16,6 +16,9 @@ type EvaluationConfiguration struct { Models ModelsConfiguration // Repositories holds repository configuration data. Repositories RepositoryConfiguration + + // APIRequestAttempts holds the number of API request attempts. + APIRequestAttempts int } // ModelsConfiguration holds model data of how an evaluation was configured. diff --git a/cmd/eval-dev-quality/cmd/evaluate.go b/cmd/eval-dev-quality/cmd/evaluate.go index a7ff2cab..ad5d09dc 100644 --- a/cmd/eval-dev-quality/cmd/evaluate.go +++ b/cmd/eval-dev-quality/cmd/evaluate.go @@ -145,6 +145,7 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate. command.ModelIDsWithProviderAndAttributes = configuration.Models.Selected command.Repositories = configuration.Repositories.Selected + command.APIRequestAttempts = uint(configuration.APIRequestAttempts) } // Check and validate common options. @@ -173,6 +174,7 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate. } evaluationContext.APIReqestAttempts = command.APIRequestAttempts evaluationContext.APIRequestTimeout = command.APIRequestTimeout + evaluationConfiguration.APIRequestAttempts = int(command.APIRequestAttempts) if command.ExecutionTimeout == 0 { command.logger.Panicf("execution timeout for compilation and tests must be greater than zero") diff --git a/cmd/eval-dev-quality/cmd/evaluate_test.go b/cmd/eval-dev-quality/cmd/evaluate_test.go index 97829e75..2579677d 100644 --- a/cmd/eval-dev-quality/cmd/evaluate_test.go +++ b/cmd/eval-dev-quality/cmd/evaluate_test.go @@ -1448,7 +1448,8 @@ func TestEvaluateInitialize(t *testing.T) { "golang/plain", "java/plain" ] - } + }, + "APIRequestAttempts": 1 } `) require.NoError(t, os.WriteFile(filepath.Join(workingDirectory, "config.json"), []byte(configurationContent), 0700))