Merge pull request #3295 from buildkite/ming/te-3708-follow-up

TE-3708-follow-up: Use go test -cover to generate coverage report

Author: zhming0

.buildkite/pipeline.yml

@@ -23,7 +23,9 @@ steps:
     key: test-linux-amd64
     command: ".buildkite/steps/tests.sh"
     parallelism: 2
-    artifact_paths: junit-*.xml
+    artifact_paths:
+      - junit-*.xml
+      - "coverage/**/*"
     plugins:
       - docker-compose#v4.14.0:
           config: .buildkite/docker-compose.yml
@@ -33,14 +35,14 @@ steps:
       - test-collector#v1.2.0:
           files: "junit-*.xml"
           format: "junit"
-      - artifacts#v1.9.0:
-          upload: "cover.{html,out}"
 
   - name: ":linux: Linux ARM64 Tests"
     key: test-linux-arm64
     command: ".buildkite/steps/tests.sh"
     parallelism: 2
-    artifact_paths: junit-*.xml
+    artifact_paths:
+      - junit-*.xml
+      - "coverage/**/*"
     agents:
       queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
     plugins:
@@ -52,15 +54,15 @@ steps:
       - test-collector#v1.2.0:
           files: "junit-*.xml"
           format: "junit"
-      - artifacts#v1.9.0:
-          upload: "cover.{html,out}"
 
   - name: ":satellite: Detect Data Races"
     key: test-race-linux-arm64
     command: ".buildkite/steps/tests.sh -race"
     # Extra parallelism because this data race test is slow
     parallelism: 3
-    artifact_paths: junit-*.xml
+    artifact_paths:
+      - junit-*.xml
+      - "coverage/**/*"
     agents:
       queue: $AGENT_RUNNERS_LINUX_ARM64_QUEUE
     plugins:
@@ -72,22 +74,71 @@ steps:
       - test-collector#v1.2.0:
           files: "junit-*.xml"
           format: "junit"
-      - artifacts#v1.9.0:
-          upload: "cover.{html,out}"
 
   - name: ":windows: Windows AMD64 Tests"
     key: test-windows
     command: "bash .buildkite\\steps\\tests.sh"
     parallelism: 2
-    artifact_paths: junit-*.xml
+    artifact_paths:
+      - junit-*.xml
+      - "coverage/**/*"
     agents:
       queue: $AGENT_RUNNERS_WINDOWS_QUEUE
     plugins:
       - test-collector#v1.2.0:
           files: "junit-*.xml"
           format: "junit"
-      - artifacts#v1.9.0:
-          upload: "cover.{html,out}"
+
+  - name: ":coverage: Test coverage report Linux ARM64"
+    key: test-coverage-linux-arm64
+    command: ".buildkite/steps/test-coverage-report.sh"
+    artifact_paths:
+      - "cover.html"
+      - "cover.out"
+    depends_on:
+      - test-linux-arm64
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          run: agent
+      - artifacts#v1.9.4:
+          download: "coverage/**"
+          step: test-linux-arm64
+
+  - name: ":coverage: Test coverage report Linux AMD64"
+    key: test-coverage-linux-amd64
+    command: ".buildkite/steps/test-coverage-report.sh"
+    artifact_paths:
+      - "cover.html"
+      - "cover.out"
+    depends_on:
+      - test-linux-amd64
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          run: agent
+      - artifacts#v1.9.4:
+          download: "coverage/**"
+          step: test-linux-amd64
+
+  - name: ":coverage: Test coverage report Linux ARM64 Race"
+    key: test-coverage-linux-arm64-race
+    command: ".buildkite/steps/test-coverage-report.sh"
+    artifact_paths:
+      - "cover.html"
+      - "cover.out"
+    depends_on:
+      - test-race-linux-arm64
+    plugins:
+      - docker-compose#v4.14.0:
+          config: .buildkite/docker-compose.yml
+          cli-version: 2
+          run: agent
+      - artifacts#v1.9.4:
+          download: "coverage/**"
+          step: test-race-linux-arm64
 
   - label: ":writing_hand: Annotate with Test Failures"
     depends_on:

.buildkite/steps/test-coverage-report.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo 'Producing coverage report'
+go tool covdata textfmt -i "coverage" -o cover.out
+go tool cover -html cover.out -o cover.html

.buildkite/steps/tests.sh

@@ -8,14 +8,18 @@ go install gotest.tools/gotestsum@v1.8.0
 
 go install github.com/buildkite/test-engine-client@v1.5.0-rc.3
 
-echo '+++ Running tests'
 export BUILDKITE_TEST_ENGINE_SUITE_SLUG=buildkite-agent
 export BUILDKITE_TEST_ENGINE_TEST_RUNNER=gotest
 export BUILDKITE_TEST_ENGINE_RESULT_PATH="junit-${BUILDKITE_JOB_ID}.xml"
 export BUILDKITE_TEST_ENGINE_RETRY_COUNT=1
-export BUILDKITE_TEST_ENGINE_TEST_CMD="gotestsum --junitfile={{resultPath}} -- -count=1 -coverprofile=cover.out $@ {{packages}}"
+if [[ "$(go env GOOS)" == "windows" ]]; then
+  # I can't get windows to work with the $COVERAGE_DIR, I tried cygpath but no luck.
+  # need a Windows VM to debug.
+  export BUILDKITE_TEST_ENGINE_TEST_CMD="gotestsum --junitfile={{resultPath}} -- -count=1 $* {{packages}}"
+else
+  mkdir -p coverage
+  COVERAGE_DIR="$PWD/coverage"
+  export BUILDKITE_TEST_ENGINE_TEST_CMD="gotestsum --junitfile={{resultPath}} -- -count=1 -cover $* {{packages}} -test.gocoverdir=${COVERAGE_DIR}"
+fi
 
 test-engine-client
-
-echo 'Producing coverage report'
-go tool cover -html cover.out -o cover.html