Skip to content

Commit 01c2262

Browse files
bauersimonbauersimon
committed
TODO Track coverage for Rust tests
1 parent b9ce620 commit 01c2262

File tree

3 files changed

+207
-13
lines changed

3 files changed

+207
-13
lines changed

‎evaluate/task/test-integration/task_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func TestWriteTestsRun(t *testing.T) {
137137
evaluatetask.IdentifierWriteTests: metrics.Assessments{
138138
metrics.AssessmentKeyGenerateTestsForFileCharacterCount: 84,
139139
metrics.AssessmentKeyResponseCharacterCount: 98,
140-
metrics.AssessmentKeyCoverage: 0, // TODO Get coverage.
140+
metrics.AssessmentKeyCoverage: 3,
141141
metrics.AssessmentKeyFilesExecuted: 1,
142142
metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
143143
metrics.AssessmentKeyResponseNoError: 1,
@@ -147,7 +147,7 @@ func TestWriteTestsRun(t *testing.T) {
147147
evaluatetask.IdentifierWriteTestsSymflowerFix: metrics.Assessments{
148148
metrics.AssessmentKeyGenerateTestsForFileCharacterCount: 84,
149149
metrics.AssessmentKeyResponseCharacterCount: 98,
150-
metrics.AssessmentKeyCoverage: 0, // TODO Get coverage.
150+
metrics.AssessmentKeyCoverage: 3,
151151
metrics.AssessmentKeyFilesExecuted: 1,
152152
metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
153153
metrics.AssessmentKeyResponseNoError: 1,
@@ -157,7 +157,7 @@ func TestWriteTestsRun(t *testing.T) {
157157
evaluatetask.IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{
158158
metrics.AssessmentKeyGenerateTestsForFileCharacterCount: 84,
159159
metrics.AssessmentKeyResponseCharacterCount: 98,
160-
metrics.AssessmentKeyCoverage: 0, // TODO Get coverage.
160+
metrics.AssessmentKeyCoverage: 3,
161161
metrics.AssessmentKeyFilesExecuted: 1,
162162
metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
163163
metrics.AssessmentKeyResponseNoError: 1,
@@ -167,7 +167,7 @@ func TestWriteTestsRun(t *testing.T) {
167167
evaluatetask.IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{
168168
metrics.AssessmentKeyGenerateTestsForFileCharacterCount: 84,
169169
metrics.AssessmentKeyResponseCharacterCount: 98,
170-
metrics.AssessmentKeyCoverage: 0, // TODO Get coverage.
170+
metrics.AssessmentKeyCoverage: 3,
171171
metrics.AssessmentKeyFilesExecuted: 1,
172172
metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
173173
metrics.AssessmentKeyResponseNoError: 1,

‎language/rust/language.go‎

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package rust
22

33
import (
4+
"bufio"
45
"context"
56
"os"
67
"path/filepath"
@@ -71,12 +72,57 @@ func (l *Language) DefaultTestFileSuffix() string {
7172
return ".rs"
7273
}
7374

75+
// testDirectiveLinePerFile computes the line of the "#[cfg(test)]" compiler directive within each file.
76+
// Lines are counted index-1 based.
77+
func (l *Language) testDirectiveLinePerFile(logger *log.Logger, repositoryPath string) (linePerFile map[string]int, err error) {
78+
files, err := l.Files(logger, repositoryPath)
79+
if err != nil {
80+
return nil, err
81+
}
82+
83+
linePerFile = map[string]int{}
84+
for _, filePath := range files {
85+
file, err := os.Open(filepath.Join(repositoryPath, filePath))
86+
if err != nil {
87+
return nil, pkgerrors.WithStack(err)
88+
}
89+
defer file.Close()
90+
91+
line := 0
92+
scanner := bufio.NewScanner(file)
93+
for scanner.Scan() {
94+
if err := scanner.Err(); err != nil {
95+
return nil, pkgerrors.WithStack(err)
96+
}
97+
line++
98+
if strings.Contains(scanner.Text(), "#[cfg(test)]") {
99+
linePerFile[filePath] = line
100+
101+
break
102+
}
103+
}
104+
}
105+
106+
return linePerFile, nil
107+
}
108+
74109
// ExecuteTests invokes the language specific testing on the given repository.
75110
func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (testResult *language.TestResult, problems []error, err error) {
76-
commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
77-
Command: []string{ // TODO Move this to `symflower test` to get coverage information.
78-
"cargo",
79-
"llvm-cov",
111+
// HACK Tests in Rust are within the implementation file, but excluding coverage more granular than file-level is currently unstable (https://github.com/rust-lang/rust/issues/84605). Therefore, we assume that test are always at the end of the file and ignore reported coverage after the "#[cfg(test)]" compiler directive.
112+
testStartLinePerFile, err := l.testDirectiveLinePerFile(logger, repositoryPath)
113+
if err != nil {
114+
return nil, nil, err
115+
}
116+
117+
ctx, cancel := context.WithTimeout(context.Background(), language.DefaultExecutionTimeout)
118+
defer cancel()
119+
coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
120+
commandOutput, err := util.CommandWithResult(ctx, logger, &util.Command{
121+
Command: []string{
122+
"symflower-local", "test", // TODO set to symflower default tool path once released
123+
"--language", "rust",
124+
"--workspace", repositoryPath,
125+
"--coverage-file", coverageFilePath,
80126
},
81127

82128
Directory: repositoryPath,
@@ -105,11 +151,15 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test
105151
StdOut: commandOutput,
106152
}
107153

108-
// coverageFilePath := "" // TODO Get coverage information.
109-
// testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath)
110-
// if err != nil {
111-
// return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
112-
// }
154+
coverageData, err := language.ParseCoverage(logger, coverageFilePath)
155+
if err != nil {
156+
return testResult, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
157+
}
158+
for _, block := range coverageData {
159+
if lineStart, ok := testStartLinePerFile[block.FilePath]; ok && block.LineEnd < lineStart && block.Count > 0 {
160+
testResult.Coverage++
161+
}
162+
}
113163

114164
return testResult, problems, nil
115165
}

‎language/rust/language_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package rust
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
"github.com/symflower/eval-dev-quality/log"
11+
"github.com/zimmski/osutil/bytesutil"
12+
)
13+
14+
func TestTestDirectiveLinePerFile(t *testing.T) {
15+
type testCase struct {
16+
Name string
17+
18+
Files map[string]string
19+
20+
Expected map[string]int
21+
}
22+
23+
validate := func(t *testing.T, tc *testCase) {
24+
t.Run(tc.Name, func(t *testing.T) {
25+
buffer, logger := log.Buffer()
26+
defer func() {
27+
if t.Failed() {
28+
t.Logf("Logs:%s", buffer.String())
29+
}
30+
}()
31+
32+
directory := t.TempDir()
33+
for path, content := range tc.Files {
34+
require.NoError(t, os.WriteFile(
35+
filepath.Join(directory, path),
36+
[]byte(bytesutil.StringTrimIndentations(content)),
37+
0700,
38+
))
39+
}
40+
41+
actual, err := (&Language{}).testDirectiveLinePerFile(logger, directory)
42+
require.NoError(t, err)
43+
assert.Equal(t, tc.Expected, actual)
44+
})
45+
}
46+
47+
validate(t, &testCase{
48+
Name: "No tests",
49+
50+
Files: map[string]string{
51+
"main.rs": `
52+
fn main() {}
53+
`,
54+
},
55+
56+
Expected: map[string]int{},
57+
})
58+
validate(t, &testCase{
59+
Name: "Single test",
60+
61+
Files: map[string]string{
62+
"plain.rs": `
63+
pub fn plain() {
64+
// This does not do anything but it gives us a line to cover.
65+
}
66+
67+
#[cfg(test)]
68+
mod tests {
69+
use super::*;
70+
71+
#[test]
72+
fn test_plain() {
73+
// Simply call the function to ensure it runs without panicking
74+
plain();
75+
// Since plain() doesn't return anything or have observable side effects,
76+
// we can only verify that it executes without errors
77+
}
78+
}
79+
`,
80+
},
81+
82+
Expected: map[string]int{
83+
"plain.rs": 5,
84+
},
85+
})
86+
validate(t, &testCase{
87+
Name: "Multiple tests",
88+
89+
Files: map[string]string{
90+
"plain.rs": `
91+
pub fn plain() {
92+
// This does not do anything but it gives us a line to cover.
93+
}
94+
95+
#[cfg(test)]
96+
mod tests {
97+
use super::*;
98+
99+
#[test]
100+
fn test_plain() {
101+
// Simply call the function to ensure it runs without panicking
102+
plain();
103+
// Since plain() doesn't return anything or have observable side effects,
104+
// we can only verify that it executes without errors
105+
}
106+
}
107+
`,
108+
"plain_two.rs": `
109+
pub fn plain() {
110+
// This does not do anything but it gives us a line to cover.
111+
}
112+
113+
#[cfg(test)]
114+
mod tests {
115+
use super::*;
116+
117+
#[test]
118+
fn test_plain() {
119+
// Simply call the function to ensure it runs without panicking
120+
plain();
121+
// Since plain() doesn't return anything or have observable side effects,
122+
// we can only verify that it executes without errors
123+
}
124+
}
125+
`,
126+
},
127+
128+
Expected: map[string]int{
129+
"plain.rs": 5,
130+
"plain_two.rs": 5,
131+
},
132+
})
133+
validate(t, &testCase{
134+
Name: "Non-rust file",
135+
136+
Files: map[string]string{
137+
"README.md": `
138+
Code example: #[cfg(test)]
139+
`,
140+
},
141+
142+
Expected: map[string]int{},
143+
})
144+
}

0 commit comments

Comments
 (0)