CLI flag for specifying a run ID starting index to be able to distribute runs

Part of #403

Author: Munsio

cmd/eval-dev-quality/cmd/evaluate.go

@@ -73,6 +73,8 @@ type Evaluate struct {
 	Configuration string `long:"configuration" description:"Configuration file to set up an evaluation run."`
 	// ExecutionTimeout holds the timeout for an execution.
 	ExecutionTimeout uint `long:"execution-timeout" description:"Execution timeout for compilation and tests in minutes." default:"5"`
+	// RunIDStartsAt holds the offset increment for the run id used in creating the result folders.
+	RunIDStartsAt uint `long:"run-id-starts-at" description:"Sets the starting index for the run ID." default:"1"`
 	// Runs holds the number of runs to perform.
 	Runs uint `long:"runs" description:"Number of runs to perform." default:"1"`
 	// RunsSequential indicates that interleaved runs are disabled and runs are performed sequentially.
@@ -175,6 +177,8 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 			language.DefaultExecutionTimeout = time.Duration(command.ExecutionTimeout) * time.Minute
 		}
 
+		evaluationContext.RunIDStartsAt = command.RunIDStartsAt
+
 		if command.Runs == 0 {
 			command.logger.Panicf("number of configured runs must be greater than zero")
 		}

evaluate/evaluate.go

@@ -38,6 +38,8 @@ type Context struct {
 	// TestdataPath determines the testdata path where all repositories reside grouped by languages.
 	TestdataPath string
 
+	// RunIDStartsAt holds the run ID starting index created when running a evaluation multiple times.
+	RunIDStartsAt uint
 	// Runs holds the number of runs to perform.
 	Runs uint
 	// RunsSequential indicates that interleaved runs are disabled and runs are performed sequentially.
@@ -148,6 +150,7 @@ func Evaluate(ctx *Context) {
 								} else {
 									runCount = rl + 1
 								}
+								runID := ctx.RunIDStartsAt + runCount - 1
 
 								if err := temporaryRepository.Reset(logger); err != nil {
 									logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
@@ -177,7 +180,7 @@ func Evaluate(ctx *Context) {
 								}
 
 								// Write the task assessment to the evaluation CSV file.
-								if err := evaluationFile.WriteEvaluationRecord(model, language, temporaryRepository.Name(), runCount, assessment); err != nil {
+								if err := evaluationFile.WriteEvaluationRecord(model, language, temporaryRepository.Name(), runID, assessment); err != nil {
 									logger.Panicf("ERROR: cannot write evaluation record: %s", err)
 								}
 							}
@@ -271,6 +274,7 @@ func Evaluate(ctx *Context) {
 								} else {
 									runCount = rl + 1
 								}
+								runID := ctx.RunIDStartsAt + runCount - 1
 
 								if err := temporaryRepository.Reset(logger); err != nil {
 									logger.Panicf("ERROR: unable to reset temporary repository path: %s", err)
@@ -292,7 +296,7 @@ func Evaluate(ctx *Context) {
 								}
 
 								// Write the task assessment to the evaluation CSV file.
-								if err := evaluationFile.WriteEvaluationRecord(model, language, temporaryRepository.Name(), runCount, assessment); err != nil {
+								if err := evaluationFile.WriteEvaluationRecord(model, language, temporaryRepository.Name(), runID, assessment); err != nil {
 									logger.Panicf("ERROR: cannot write evaluation record: %s", err)
 								}
 							}

evaluate/evaluate_test.go

@@ -29,6 +29,8 @@ import (
 	modeltesting "github.com/symflower/eval-dev-quality/model/testing"
 	"github.com/symflower/eval-dev-quality/provider"
 	providertesting "github.com/symflower/eval-dev-quality/provider/testing"
+	"github.com/symflower/gota/dataframe"
+	"github.com/symflower/gota/series"
 )
 
 var (
@@ -1235,6 +1237,7 @@ func TestEvaluate(t *testing.T) {
 						repositoryPath,
 					},
 
+					RunIDStartsAt:  11,
 					Runs:           3,
 					RunsSequential: false,
 				},
@@ -1395,7 +1398,14 @@ func TestEvaluate(t *testing.T) {
 						assert.Equal(t, 1, strings.Count(data, "creating temporary repository"), "create only one temporary repository")
 					},
 					filepath.Join(string(evaluatetask.IdentifierWriteTests), log.CleanModelNameForFileSystem(mockedModelID), "golang", "golang", "plain", "evaluation.log"): nil,
-					"evaluation.csv": nil,
+					"evaluation.csv": func(t *testing.T, filePath string, data string) {
+						dataFrame := dataframe.ReadCSV(strings.NewReader(data))
+						assert.NoError(t, dataFrame.Err)
+
+						expectedColumnRun := series.New([]int{11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13}, series.Int, "run")
+						actualColumnRun := dataFrame.Col("run")
+						assert.Equal(t, expectedColumnRun, actualColumnRun)
+					},
 				},
 			})
 		}
@@ -1425,6 +1435,7 @@ func TestEvaluate(t *testing.T) {
 						repositoryPath,
 					},
 
+					RunIDStartsAt:  21,
 					Runs:           3,
 					RunsSequential: true,
 				},
@@ -1585,7 +1596,14 @@ func TestEvaluate(t *testing.T) {
 						assert.Contains(t, data, "\"msg\":\"starting run\",\"count\":3,\"total\":3,")
 						assert.NotRegexp(t, `\\\"msg\\\":\\\"starting run\\\",\\\"count\\\":\d+,\\\"total\\\":\d+\}`, data)
 					},
-					"evaluation.csv": nil,
+					"evaluation.csv": func(t *testing.T, filePath string, data string) {
+						dataFrame := dataframe.ReadCSV(strings.NewReader(data))
+						assert.NoError(t, dataFrame.Err)
+
+						expectedColumnRun := series.New([]int{21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23}, series.Int, "run")
+						actualColumnRun := dataFrame.Col("run")
+						assert.Equal(t, expectedColumnRun, actualColumnRun)
+					},
 				},
 			})
 		}