TODO, Combined changes to test cases because of logging changes

Sorry but I won't split these...

Author: bauersimon

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -462,7 +462,7 @@ func TestEvaluateExecute(t *testing.T) {
 
 				ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
 					filepath.Join("result-directory", "evaluation.log"): func(t *testing.T, filePath string, data string) {
-						assert.Contains(t, data, "Skipping unavailable provider \"openrouter\"")
+						assert.Contains(t, data, "Skipping unavailable provider \\\"openrouter\\\"")
 					},
 				},
 				ExpectedPanicContains: "ERROR: model openrouter/auto does not exist",

evaluate/task/repository_test.go

@@ -204,7 +204,7 @@ func TestRepositoryLoadConfiguration(t *testing.T) {
 		TestDataPath:   filepath.Join("..", "..", "testdata"),
 		RepositoryPath: filepath.Join("golang", "plain"),
 
-		ExpectedErrorText: "task identifier \"unknown-task\" unknown",
+		ExpectedErrorText: "task identifier \\\"unknown-task\\\" unknown",
 		MutationBefore: func(t *testing.T, repositoryPath string) {
 			configuration := bytesutil.StringTrimIndentations(`
 				{

evaluate/task/test-integration/task_test.go

@@ -85,10 +85,10 @@ func TestWriteTestsRun(t *testing.T) {
 			},
 		},
 		ValidateLog: func(t *testing.T, data string) {
-			assert.Contains(t, data, "Evaluating model \"symflower/symbolic-execution\"")
+			assert.Contains(t, data, "Evaluating model \\\"symflower/symbolic-execution\\\"")
 			assert.Contains(t, data, "Generated 1 test")
 			assert.Contains(t, data, "PASS: TestSymflowerPlain")
-			assert.Contains(t, data, "Evaluated model \"symflower/symbolic-execution\"")
+			assert.Contains(t, data, "Evaluated model \\\"symflower/symbolic-execution\\\"")
 		},
 	})
 }

evaluate/task/transpile_test.go

@@ -662,7 +662,7 @@ func TestValidateTranspileRepository(t *testing.T) {
 		Language:       &golang.Language{},
 
 		ExpectedError: func(t *testing.T, err error) {
-			assert.ErrorContains(t, err, "the language extension \".unsupported\" is not supported")
+			assert.ErrorContains(t, err, "the language extension \\\".unsupported\\\" is not supported")
 		},
 	})
 	t.Run("Go", func(t *testing.T) {

evaluate/task/write-test_test.go

@@ -53,7 +53,7 @@ func TestWriteTestsRun(t *testing.T) {
 		// Generate invalid code for caseA (with and without template).
 		modelMock.RegisterGenerateSuccess(t, "caseA_test.go", "does not compile", metricstesting.AssessmentsWithProcessingTime).Twice()
 		// Generate valid code for caseB (with and without template).
-		modelMock.RegisterGenerateSuccess(t, "caseB_test.go", "package plain\n\nimport \"testing\"\n\nfunc TestCaseB(t *testing.T){}", metricstesting.AssessmentsWithProcessingTime).Twice()
+		modelMock.RegisterGenerateSuccess(t, "caseB_test.go", "package plain\n\nimport \\\"testing\\\"\n\nfunc TestCaseB(t *testing.T){}", metricstesting.AssessmentsWithProcessingTime).Twice()
 
 		validate(t, &tasktesting.TestCaseTask{
 			Name: "Plain",
@@ -93,7 +93,7 @@ func TestWriteTestsRun(t *testing.T) {
 				"exit status 1",                  // Symflower fix not applicable (overwrote template).
 			},
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Equal(t, 1, strings.Count(data, "Evaluating model \"mocked-model\""))
+				assert.Equal(t, 1, strings.Count(data, "Evaluating model \\\"mocked-model\\\""))
 				assert.Equal(t, 4, strings.Count(data, "PASS: TestCaseB")) // Bare model result, with fix, with template, with template and fix.
 			},
 		})
@@ -120,7 +120,7 @@ func TestWriteTestsRun(t *testing.T) {
 					ExpectedRepositoryAssessment: expectedAssessments,
 					ExpectedProblemContains:      expectedProblems,
 					ValidateLog: func(t *testing.T, data string) {
-						assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+						assert.Contains(t, data, "Evaluating model \\\"mocked-model\\\"")
 						if assertTestsPass {
 							assert.Contains(t, data, "PASS: TestPlain")
 						}
@@ -295,7 +295,7 @@ func TestWriteTestsRun(t *testing.T) {
 			},
 			ExpectedProblemContains: nil,
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Contains(t, data, "Evaluating model \"mocked-model\"")
+				assert.Contains(t, data, "Evaluating model \\\"mocked-model\\\"")
 			},
 		})
 	}
@@ -390,7 +390,7 @@ func TestWriteTestsRun(t *testing.T) {
 				IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{},
 			},
 			ValidateLog: func(t *testing.T, data string) {
-				assert.Contains(t, data, "Ignoring file \"plain.go\" (as configured by the repository)")
+				assert.Contains(t, data, "Ignoring file \\\"plain.go\\\" (as configured by the repository)")
 			},
 		})
 	}

tools/install_test.go

@@ -36,7 +36,7 @@ func ValidateInstallTool(t *testing.T, tool Tool) {
 		logOutput, logger := log.Buffer()
 		require.NoError(t, InstallTool(logger, tool, temporaryPath))
 
-		require.Contains(t, logOutput.String(), fmt.Sprintf(`Install %q to`, tool.ID()))
+		require.Contains(t, logOutput.String(), fmt.Sprintf(`msg="installing tool" tool=%s`, tool.ID()))
 		toolPath, err := exec.LookPath(tool.BinaryName())
 		require.NoError(t, err)
 		require.NotEmpty(t, toolPath)
@@ -46,7 +46,7 @@ func ValidateInstallTool(t *testing.T, tool Tool) {
 		logOutput, logger := log.Buffer()
 		require.NoError(t, InstallTool(logger, tool, temporaryPath))
 
-		require.NotContains(t, logOutput.String(), fmt.Sprintf(`Install %q to`, tool.ID()))
+		require.NotContains(t, logOutput.String(), fmt.Sprintf(`msg="installing tool" tool=%s`, tool.ID()))
 		toolPath, err := exec.LookPath(tool.BinaryName())
 		require.NoError(t, err)
 		require.NotEmpty(t, toolPath)