refactor, Clarify API request attempts to conform to our terminology

Author: bauersimon

cmd/eval-dev-quality/cmd/evaluate.go

@@ -59,10 +59,10 @@ type Evaluate struct {
 	ProviderTokens map[string]string `long:"tokens" description:"API tokens for model providers (of the form '$provider:$token'). When using the environment variable, separate multiple definitions with ','." env:"PROVIDER_TOKEN" env-delim:","`
 	// ProviderUrls holds all custom inference endpoint urls for the providers.
 	ProviderUrls map[string]string `long:"urls" description:"Custom OpenAI API compatible inference endpoints (of the form '$provider:$url,...'). Use '$provider=custom-$name' to manually register a custom OpenAI API endpoint provider. Note that the models of a custom OpenAI API endpoint provider must be declared explicitly using the '--model' option. When using the environment variable, separate multiple definitions with ','." env:"PROVIDER_URL" env-delim:","`
-	// QueryAttempts holds the number of query attempts to perform when a model request errors in the process of solving a task.
-	QueryAttempts uint `long:"query-attempts" description:"Number of query attempts to perform when a model request errors in the process of solving a task." default:"3"`
-	// QueryTimeout holds the timeout for model requests.
-	QueryTimeout uint `long:"query-timeout" description:"Timeout of a model query in seconds. ('0' to disable)" default:"1200"`
+	// APIRequestAttempts holds the number of allowed API requests per LLM query.
+	APIRequestAttempts uint `long:"api-request-attempts" description:"Number of allowed API requests per LLM query." default:"3"`
+	// APIRequestTimeout holds the timeout for API requests in seconds.
+	APIRequestTimeout uint `long:"api-request-timeout" description:"Timeout of API requests in seconds. ('0' to disable)" default:"1200"`
 
 	// Repositories determines which repository should be used for the evaluation, or empty if all repositories should be used.
 	Repositories []string `long:"repository" description:"Evaluate with this repository. By default all repositories are used."`
@@ -168,11 +168,11 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 			tools.SymflowerPath = command.SymflowerBinaryPath
 		}
 
-		if command.QueryAttempts == 0 {
-			command.logger.Panicf("number of configured query attempts must be greater than zero")
+		if command.APIRequestAttempts == 0 {
+			command.logger.Panicf("number of configured API requests must be greater than zero")
 		}
-		evaluationContext.QueryAttempts = command.QueryAttempts
-		evaluationContext.QueryTimeout = command.QueryTimeout
+		evaluationContext.APIReqestAttempts = command.APIRequestAttempts
+		evaluationContext.APIRequestTimeout = command.APIRequestTimeout
 
 		if command.ExecutionTimeout == 0 {
 			command.logger.Panicf("execution timeout for compilation and tests must be greater than zero")

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -1272,7 +1272,7 @@ func TestEvaluateInitialize(t *testing.T) {
 	makeValidCommand := func(modify func(command *Evaluate)) *Evaluate {
 		c := &Evaluate{
 			ModelIDsWithProviderAndAttributes: []string{"symflower/smart-template"},
-			QueryAttempts:                     1,
+			APIRequestAttempts:                1,
 
 			ResultPath:   filepath.Join("$TEMP_PATH", "result-directory"),
 			TestdataPath: filepath.Join("..", "..", "..", "testdata"),
@@ -1530,7 +1530,7 @@ func TestEvaluateInitialize(t *testing.T) {
 		Name: "Attempts parameter hast to be greater then zero",
 
 		Command: makeValidCommand(func(command *Evaluate) {
-			command.QueryAttempts = 0
+			command.APIRequestAttempts = 0
 		}),
 
 		ValidatePanic: "number of configured query attempts must be greater than zero",

evaluate/evaluate.go

@@ -28,10 +28,10 @@ type Context struct {
 	Models []evalmodel.Model
 	// ProviderForModel holds the models and their associated provider.
 	ProviderForModel map[evalmodel.Model]provider.Provider
-	// QueryAttempts holds the number of query attempts to perform when a model request errors in the process of solving a task.
-	QueryAttempts uint
-	// QueryTimeout holds the timeout for model queries in seconds.
-	QueryTimeout uint
+	// APIReqestAttempts holds the number of allowed API requests per LLM query.
+	APIReqestAttempts uint
+	// APIRequestTimeout holds the timeout for API requests in seconds.
+	APIRequestTimeout uint
 
 	// RepositoryPaths determines which relative repository paths should be used for the evaluation, or empty if all repositories should be used.
 	RepositoryPaths []string
@@ -132,9 +132,9 @@ func Evaluate(ctx *Context) {
 						modelSucceededBasicChecksOfLanguage[model] = map[evallanguage.Language]bool{}
 					}
 
-					if r, ok := model.(evalmodel.SetQueryHandling); ok {
-						r.SetQueryAttempts(ctx.QueryAttempts)
-						r.SetQueryTimeout(ctx.QueryTimeout)
+					if r, ok := model.(evalmodel.ConfigureAPIRequestHandling); ok {
+						r.SetAPIRequestAttempts(ctx.APIReqestAttempts)
+						r.SetAPIRequestTimeout(ctx.APIRequestTimeout)
 					}
 
 					for _, taskIdentifier := range temporaryRepository.Configuration().Tasks {

evaluate/evaluate_test.go

@@ -105,8 +105,8 @@ func TestEvaluate(t *testing.T) {
 			logger = logger.With(log.AttributeKeyResultPath, resultPath)
 
 			tc.Context.Log = logger
-			if tc.Context.QueryAttempts == 0 {
-				tc.Context.QueryAttempts = 1
+			if tc.Context.APIReqestAttempts == 0 {
+				tc.Context.APIReqestAttempts = 1
 			}
 			tc.Context.ResultPath = resultPath
 			if tc.Context.TestdataPath == "" {
@@ -206,7 +206,7 @@ func TestEvaluate(t *testing.T) {
 				Models: []evalmodel.Model{
 					mockedModel,
 				},
-				QueryAttempts: 3,
+				APIReqestAttempts: 3,
 
 				RepositoryPaths: []string{
 					repositoryPath,
@@ -310,7 +310,7 @@ func TestEvaluate(t *testing.T) {
 					Models: []evalmodel.Model{
 						mockedModel,
 					},
-					QueryAttempts: 1,
+					APIReqestAttempts: 1,
 				},
 
 				ExpectedAssessments: []*metricstesting.AssessmentTuple{
@@ -397,7 +397,7 @@ func TestEvaluate(t *testing.T) {
 					Models: []evalmodel.Model{
 						mockedModel,
 					},
-					QueryAttempts: 3,
+					APIReqestAttempts: 3,
 
 					RepositoryPaths: []string{
 						repositoryPath,
@@ -500,7 +500,7 @@ func TestEvaluate(t *testing.T) {
 					Models: []evalmodel.Model{
 						mockedModel,
 					},
-					QueryAttempts: 3,
+					APIReqestAttempts: 3,
 
 					RepositoryPaths: []string{
 						repositoryPath,

model/llm/llm.go

@@ -33,10 +33,10 @@ type Model struct {
 
 	// attributes holds query attributes.
 	attributes map[string]string
-	// queryAttempts holds the number of query attempts to perform when a model request errors in the process of solving a task.
-	queryAttempts uint
-	// queryTimeout holds the timeout for model requests in seconds.
-	queryTimeout uint
+	// apiRequestAttempts holds the number of allowed API requests per LLM query.
+	apiRequestAttempts uint
+	// apiRequestTimeout holds the timeout for API requests in seconds.
+	apiRequestTimeout uint
 
 	// metaInformation holds a model meta information.
 	metaInformation *model.MetaInformation
@@ -48,8 +48,8 @@ func NewModel(provider provider.Query, modelIDWithAttributes string) (llmModel *
 		id:       modelIDWithAttributes,
 		provider: provider,
 
-		queryAttempts: 1,
-		queryTimeout:  0,
+		apiRequestAttempts: 1,
+		apiRequestTimeout:  0,
 	}
 	llmModel.modelID, llmModel.attributes = model.ParseModelID(modelIDWithAttributes)
 
@@ -63,8 +63,8 @@ func NewModelWithMetaInformation(provider provider.Query, modelIdentifier string
 		provider: provider,
 		modelID:  modelIdentifier,
 
-		queryAttempts: 1,
-		queryTimeout:  0,
+		apiRequestAttempts: 1,
+		apiRequestTimeout:  0,
 
 		metaInformation: metaInformation,
 	}
@@ -338,8 +338,8 @@ func (m *Model) query(logger *log.Logger, request string) (queryResult *provider
 		func() error {
 			logger.Info("querying model", "model", m.ID(), "query-id", id, "prompt", string(bytesutil.PrefixLines([]byte(request), []byte("\t"))))
 			ctx := context.Background()
-			if m.queryTimeout > 0 {
-				c, cancel := context.WithTimeoutCause(ctx, time.Second*time.Duration(m.queryTimeout), pkgerrors.Errorf("request query timeout (%d seconds)", m.queryTimeout))
+			if m.apiRequestTimeout > 0 {
+				c, cancel := context.WithTimeoutCause(ctx, time.Second*time.Duration(m.apiRequestTimeout), pkgerrors.Errorf("API request timed out (%d seconds)", m.apiRequestTimeout))
 				defer cancel()
 				ctx = c
 			}
@@ -360,12 +360,12 @@ func (m *Model) query(logger *log.Logger, request string) (queryResult *provider
 
 			return nil
 		},
-		retry.Attempts(m.queryAttempts),
+		retry.Attempts(m.apiRequestAttempts),
 		retry.Delay(5*time.Second),
 		retry.DelayType(retry.BackOffDelay),
 		retry.LastErrorOnly(true),
 		retry.OnRetry(func(n uint, err error) {
-			logger.Info("query retry", "count", n+1, "total", m.queryAttempts, "error", err)
+			logger.Info("API request attempt failed", "count", n+1, "total", m.apiRequestAttempts, "error", err)
 		}),
 	); err != nil {
 		return nil, err
@@ -533,14 +533,14 @@ func handleQueryResult(queryResult *provider.QueryResult, filePathAbsolute strin
 	return assessment, nil
 }
 
-var _ model.SetQueryHandling = (*Model)(nil)
+var _ model.ConfigureAPIRequestHandling = (*Model)(nil)
 
-// SetQueryAttempts sets the number of query attempts to perform when a model request errors in the process of solving a task.
-func (m *Model) SetQueryAttempts(queryAttempts uint) {
-	m.queryAttempts = queryAttempts
+// SetAPIRequestAttempts sets the number of allowed API requests per LLM query.
+func (m *Model) SetAPIRequestAttempts(queryAttempts uint) {
+	m.apiRequestAttempts = queryAttempts
 }
 
-// SetQueryTimeout sets the timeout for model requests in seconds.
-func (m *Model) SetQueryTimeout(timeout uint) {
-	m.queryTimeout = timeout
+// SetAPIRequestTimeout sets the timeout for API requests in seconds.
+func (m *Model) SetAPIRequestTimeout(timeout uint) {
+	m.apiRequestTimeout = timeout
 }

model/llm/llm_test.go

@@ -196,8 +196,8 @@ func TestModelQuery(t *testing.T) {
 				tc.SetupMock(mock)
 			}
 			llm := NewModel(mock, "some-model")
-			llm.SetQueryAttempts(tc.QueryAttempts)
-			llm.SetQueryTimeout(tc.QueryTimeout)
+			llm.SetAPIRequestAttempts(tc.QueryAttempts)
+			llm.SetAPIRequestTimeout(tc.QueryTimeout)
 
 			queryResult, actualError := llm.query(logger, tc.Request)
 

model/model.go

@@ -82,10 +82,10 @@ type Context struct {
 	Logger *log.Logger
 }
 
-// SetQueryHandling defines a model that can configure how API queries are handled.
-type SetQueryHandling interface {
-	// SetQueryAttempts sets the number of query attempts to perform when a model request errors in the process of solving a task.
-	SetQueryAttempts(attempts uint)
-	// SetQueryTimeout sets the timeout for model requests in seconds.
-	SetQueryTimeout(timeout uint)
+// ConfigureAPIRequestHandling defines a model that can configure how API requests are handled.
+type ConfigureAPIRequestHandling interface {
+	// SetAPIRequestAttempts sets the number of allowed API requests per LLM query.
+	SetAPIRequestAttempts(attempts uint)
+	// SetAPIRequestTimeout sets the timeout for API requests in seconds.
+	SetAPIRequestTimeout(timeout uint)
 }