refactor, Do mocking setup only if test is really executed so mockito does not complain about "missing calls" when selectively targeting other tests

Author: bauersimon

evaluate/task/test-integration/task_test.go

@@ -102,28 +102,31 @@ func TestWriteTestsRun(t *testing.T) {
 	})
 	{
 		mockProvider := providertesting.NewMockQuery(t)
-		mockProvider.On("Query", mock.Anything, mock.Anything, mock.Anything).Return(
-			&provider.QueryResult{
-				Message: bytesutil.StringTrimIndentations(`
-					` + "```rust`" + `
-					#[cfg(test)]
-					mod tests {
-						use super::*;
-
-						#[test]
-						fn test_plain() {
-							plain();
-						}
-					}
-					` + "```" + `
-				`),
-			},
-			nil,
-		).After(100 * time.Millisecond)
 		model := llm.NewModel(mockProvider, "model")
 		validate(t, &tasktesting.TestCaseTask{
 			Name: "Rust",
 
+			Setup: func(t *testing.T) {
+				mockProvider.On("Query", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
+					&provider.QueryResult{
+						Message: bytesutil.StringTrimIndentations(`
+							` + "```rust`" + `
+							#[cfg(test)]
+							mod tests {
+								use super::*;
+
+								#[test]
+								fn test_plain() {
+									plain();
+								}
+							}
+							` + "```" + `
+						`),
+					},
+					nil,
+				).After(100 * time.Millisecond)
+			},
+
 			Model:          model,
 			Language:       &rust.Language{},
 			TestDataPath:   filepath.Join("..", "..", "..", "testdata"),

evaluate/task/testing/task.go

@@ -22,6 +22,8 @@ import (
 type TestCaseTask struct {
 	Name string
 
+	Setup func(t *testing.T)
+
 	Task           evaltask.Task
 	Model          model.Model
 	Language       language.Language
@@ -51,6 +53,10 @@ func (tc *TestCaseTask) Validate(t *testing.T, createRepository createRepository
 	assert.NoError(t, err)
 	defer cleanup()
 
+	if tc.Setup != nil {
+		tc.Setup(t)
+	}
+
 	taskContext := evaltask.Context{
 		Language:   tc.Language,
 		Repository: repository,

evaluate/task/write-test_test.go

@@ -130,12 +130,15 @@ func TestWriteTestsRun(t *testing.T) {
 		require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "plain"), repositoryPath))
 
 		modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model")
-		// Simulate that a model does not generate anything.
-		modelMock.MockCapabilityWriteTests.On("WriteTests", mock.Anything).Return(metricstesting.AssessmentsWithProcessingTime, nil)
 
 		validate(t, &tasktesting.TestCaseTask{
 			Name: "Reset symflower template so it's not mistaken for model solution",
 
+			Setup: func(t *testing.T) {
+				// Simulate that a model does not generate anything.
+				modelMock.MockCapabilityWriteTests.On("WriteTests", mock.Anything).Return(metricstesting.AssessmentsWithProcessingTime, nil)
+			},
+
 			Model:          modelMock,
 			Language:       &golang.Language{},
 			TestDataPath:   temporaryDirectoryPath,
@@ -389,19 +392,22 @@ func TestWriteTestsRun(t *testing.T) {
 			// There will be no template for an empty file.
 		`)), 0666))
 		modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model")
-		modelMock.RegisterGenerateSuccess(t, "empty_test.go", "package plain\n", metricstesting.AssessmentsWithProcessingTime).Once()
-		modelMock.RegisterGenerateSuccess(t, "plain_test.go", bytesutil.StringTrimIndentations(`
-			package plain
-
-			import "testing"
-
-			func TestPlain(t *testing.T) {
-					plain()
-			}
-		`), metricstesting.AssessmentsWithProcessingTime)
 		validate(t, &tasktesting.TestCaseTask{
 			Name: "Keep non-template score if template fails",
 
+			Setup: func(t *testing.T) {
+				modelMock.RegisterGenerateSuccess(t, "empty_test.go", "package plain\n", metricstesting.AssessmentsWithProcessingTime).Once()
+				modelMock.RegisterGenerateSuccess(t, "plain_test.go", bytesutil.StringTrimIndentations(`
+					package plain
+
+					import "testing"
+
+					func TestPlain(t *testing.T) {
+							plain()
+					}
+				`), metricstesting.AssessmentsWithProcessingTime)
+			},
+
 			Model:          modelMock,
 			Language:       &golang.Language{},
 			TestDataPath:   temporaryDirectoryPath,