Skip to content

Fix graph dependencies. Improve flag coverage checks by searching docker test files. #696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix graph dependencies. Improve flag coverage checks by searching docker test files. #696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 58 additions & 16 deletions cmd/test_coverage/coverage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import (
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"

grob "github.com/MetalBlueberry/go-plotly/graph_objects"
"github.com/MetalBlueberry/go-plotly/offline"
grob "github.com/MetalBlueberry/go-plotly/generated/v2.34.0/graph_objects"
"github.com/MetalBlueberry/go-plotly/pkg/offline"
"github.com/MetalBlueberry/go-plotly/pkg/types"
"github.com/urfave/cli/v2"
"sigs.k8s.io/yaml"
)
Expand All @@ -21,9 +23,10 @@ type TestCov struct {
}

// discoverTestFiles returns a list of all the e2e files in the program directory
func discoverTestFiles(programName string) ([]string, []string, error) {
func discoverTestFiles(programName string) ([]string, []string, []string, error) {
vagrantFiles := []string{}
integrationFiles := []string{}
dockerFiles := []string{}
testRoot := filepath.Join(programName, "tests")
err := filepath.Walk(testRoot, func(path string, info os.FileInfo, err error) error {
if err != nil {
Expand All @@ -37,7 +40,18 @@ func discoverTestFiles(programName string) ([]string, []string, error) {
}
return nil
})
return vagrantFiles, integrationFiles, err
dockerTestPath := filepath.Join(testRoot, "docker")
err = filepath.Walk(dockerTestPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.HasSuffix(info.Name(), "_test.go") {
dockerFiles = append(dockerFiles, path)
}
return nil
})

return vagrantFiles, integrationFiles, dockerFiles, err
}

func extractConfigYaml(e2eFile, programPath string) (TestCov, error) {
Expand Down Expand Up @@ -167,7 +181,7 @@ func coverage(c *cli.Context) error {
if _, err := os.Stat(program); err != nil {
return fmt.Errorf("unable to find binary at %s", program)
}
e2eFiles, intTestFiles, err := discoverTestFiles(programPath)
e2eFiles, intTestFiles, dockerFiles, err := discoverTestFiles(programPath)
if err != nil {
return err
}
Expand Down Expand Up @@ -228,6 +242,26 @@ func coverage(c *cli.Context) error {
return err
}

// For docker tests, work backwards. Go through ever sever flag and search each docker test for a use of that flag.
// Instead of attempting to extract args from docker tests, as flag locations vary widely.
dockerCoverage := []TestCov{}
for _, dockerFile := range dockerFiles {
tc := TestCov{
shortPath: strings.TrimPrefix(dockerFile, programPath+"/tests/"),
serverArguments: make(map[string]bool),
}
for flag := range serverFlagSet {
b, err := os.ReadFile(dockerFile)
if err != nil {
return err
}
if strings.Contains(string(b), flag) || strings.Contains(string(b), "--"+flag+"=") {
tc.serverArguments[flag] = true
}
}
dockerCoverage = append(dockerCoverage, tc)
}

// Record covered flags and filter out invalid entries
for flag := range serverFlagSet {
for _, vC := range vagrantCoverage {
Expand All @@ -240,6 +274,11 @@ func coverage(c *cli.Context) error {
serverFlagSet[flag] += 1
}
}
for _, dC := range dockerCoverage {
if dC.serverArguments[flag] {
serverFlagSet[flag] += 1
}
}
}

totalUsedFlags := totalUsed(serverFlagSet)
Expand All @@ -263,7 +302,7 @@ func coverage(c *cli.Context) error {
fmt.Printf("Covering %d out of %d (%.2f%%) of agent flags\n", totalUsedFlags, len(agentFlagSet), percentageCover)

if c.Bool("graph") {
graphResults(serverFlagSet, vagrantCoverage, intCoverage)
graphResults(serverFlagSet, vagrantCoverage, intCoverage, dockerCoverage)
}

usedFlags := []string{}
Expand Down Expand Up @@ -322,14 +361,16 @@ func coverage(c *cli.Context) error {
return nil
}

func graphResults(serverFlagSet map[string]int, vagrantCoverage []TestCov, intCoverage []TestCov) {
data := grob.Traces{}
func graphResults(serverFlagSet map[string]int, vagrantCoverage []TestCov, intCoverage []TestCov, dockerCoverage []TestCov) {
xFlagNames := []string{}

data := []types.Trace{}
for k := range serverFlagSet {
xFlagNames = append(xFlagNames, k)
}
for _, test := range append(vagrantCoverage, intCoverage...) {
sort.Strings(xFlagNames)
allTests := append(vagrantCoverage, intCoverage...)
allTests = append(allTests, dockerCoverage...)
for _, test := range allTests {
var testGroup string
switch {
case strings.Contains(test.shortPath, "integration"):
Expand All @@ -338,6 +379,8 @@ func graphResults(serverFlagSet map[string]int, vagrantCoverage []TestCov, intCo
testGroup = "install"
case strings.Contains(test.shortPath, "e2e"):
testGroup = "e2e"
case strings.Contains(test.shortPath, "docker"):
testGroup = "docker"
}

flagHits := make([]int, len(xFlagNames))
Expand All @@ -347,11 +390,10 @@ func graphResults(serverFlagSet map[string]int, vagrantCoverage []TestCov, intCo
}
}
data = append(data, &grob.Bar{
Name: test.shortPath,
X: xFlagNames,
Y: flagHits,
Type: grob.TraceTypeBar,
Legendgroup: testGroup,
Name: types.StringType(test.shortPath),
X: types.DataArray(xFlagNames),
Y: types.DataArray(flagHits),
Legendgroup: types.StringType(testGroup),
})
}

Expand All @@ -362,7 +404,7 @@ func graphResults(serverFlagSet map[string]int, vagrantCoverage []TestCov, intCo
Text: "Server Argument Coverage",
},
Xaxis: &grob.LayoutXaxis{
Tickangle: 60,
Tickangle: types.N(40),
},
Yaxis: &grob.LayoutYaxis{
Title: &grob.LayoutYaxisTitle{
Expand Down
7 changes: 1 addition & 6 deletions cmd/test_coverage/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,6 @@ import (
var (
version = "development"
rootFlags = []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
Aliases: []string{"v"},
Usage: "verbose output",
},
&cli.BoolFlag{
Name: "graph",
Aliases: []string{"g"},
Expand Down Expand Up @@ -42,7 +37,7 @@ var (
func main() {
app := cli.NewApp()
app.Name = "test-coverage"
app.Usage = "Generate coverage report for E2E/Integration tests"
app.Usage = "Generate coverage report for E2E/Integration/Docker tests"
app.UseShortOptionHandling = true
app.Flags = rootFlags
app.Action = coverage
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
)

require (
github.com/MetalBlueberry/go-plotly v0.4.0
github.com/MetalBlueberry/go-plotly v0.7.0
github.com/aws/aws-sdk-go-v2 v1.39.6
github.com/aws/aws-sdk-go-v2/config v1.31.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
Expand Down Expand Up @@ -87,7 +87,7 @@ require (
github.com/google/go-querystring v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
Expand Down
Loading