Skip to content

Tests

Tests #2385

Workflow file for this run

name: Tests
on:
pull_request:
branches: ["**"]
merge_group:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.event.number || ((github.event_name == 'push' && github.sha) || github.ref) }}
cancel-in-progress: true
permissions: read-all
env:
# Make sure we're actually testing with the intended Go release (i.e, ensure
# no automatic toolchain download happens).
GOTOOLCHAIN: local
jobs:
##############################################################################
# Run all the code generators; and refresh the LICENSES-3rdparty.csv file
generate:
needs: coverage-preflight
runs-on: ubuntu-latest
name: Run all generators
outputs:
has-patch: ${{ steps.is-tree-dirty.outputs.result }}
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: oldstable
cache-dependency-path: "**/go.mod"
- name: Run 'go generate ./...'
run: |-
mkdir -p "${GOCOVERDIR}"
find . -name go.mod -execdir go generate ./... \;
env:
GOFLAGS: -cover -covermode=atomic -coverpkg=github.com/DataDog/orchestrion/...,./...
GOCOVERDIR: ${{ github.workspace }}/coverage
- name: Consolidate coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
run: go tool covdata textfmt -i ./coverage -o ./coverage/generator.out
- name: Determine simple go version
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
id: go
run: |-
set -euo pipefail
echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
- name: Upload coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
with:
name: coverage-generators+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
path: ./coverage/generator.out
- name: Run 'go mod tidy'
# Don't run for push, it's not necessary
if: github.event_name != 'push'
run: |-
currentTag=$(go run . version -static | cut -d' ' -f 2)
find . -iname go.mod -not -path './_docs/themes/**' -print0 | while IFS= read -r -d '' gomod; do
dir="$(dirname "${gomod}")"
if [[ -n $(go -C="${dir}" mod edit --json | jq '(.Replace // []).[] | select(.Old.Path == "github.com/DataDog/orchestrion")') ]]; then
echo "Aligning orchestrion version in ${dir} to ${currentTag} if necessary..."
go -C="${dir}" mod edit -go="$(go -C="${dir}" mod edit -json | jq -r .Go)" -require="github.com/DataDog/orchestrion@${currentTag}"
fi
go -C="${dir}" mod tidy -go="$(go -C="${dir}" mod edit -json | jq -r .Go)"
go -C="${dir}" mod edit -toolchain=none
done
- name: Refresh LICENSE-3rdparty.csv
run: ./_tools/make-licenses.sh
env:
TMPDIR: ${{ runner.temp }}
- name: Update embedded documentation
run: make docs
- name: Check if working tree is dirty
# Don't run for push, it's not necessary
if: github.event_name != 'push'
id: is-tree-dirty
run: |-
set -euxo pipefail
git add .
git status
git diff --staged --patch --exit-code > .repo.patch || echo 'result=true' >> "${GITHUB_OUTPUT}"
- name: Upload patch
if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
with:
if-no-files-found: error
include-hidden-files: true
name: repo.patch
path: .repo.patch
- name: Fail build if working tree is dirty
if: github.event_name != 'push' && steps.is-tree-dirty.outputs.result == 'true'
run: |-
echo "::error::Files have been modified by 'go generate ./...' (see logs)."
cat .repo.patch
exit 1
##############################################################################
# If the generators changed anything, and we can update the PR, then we'll
# proactively do it with the mutator token.
self-mutation:
needs: generate
runs-on: ubuntu-latest
name: Update PR with generated files
if: always() && needs.generate.outputs.has-patch == 'true' && github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.maintainer_can_modify)
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Download patch
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
with:
name: repo.patch
path: ${{ runner.temp }}
- name: Apply patch
run: |-
[ -s '${{ runner.temp }}/.repo.patch' ] && git apply '${{ runner.temp }}/.repo.patch' || echo 'Empty patch. Skipping.'
- name: Generate a GitHub token
id: generate-token
uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4
with:
app-id: ${{ vars.DD_K9_LIBRARY_GO_APP_ID }}
private-key: ${{ secrets.DD_K9_LIBRARY_GO_APP_PRIVATE_KEY }}
# We use ghcommit to create signed commits directly using the GitHub API
- name: Push changes
uses: planetscale/ghcommit-action@f24050e41f8694750427d111b52f4ef9ca81a32d # v0.2.18
with:
commit_message: "chore: update generated files"
repo: ${{ github.event.pull_request.head.repo.full_name }}
branch: ${{ github.event.pull_request.head.ref }}
env:
GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
##############################################################################
# Run the various linters we have set up...
lint:
needs: generate
runs-on: ubuntu-latest
name: Linters
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Verify license headers
run: go run ./_tools/headercheck/header_check.go
- name: vet
run: go vet ./...
- name: Check documentation for misspellings
uses: crate-ci/typos@80c8a4945eec0f6d464eaf9e65ed98ef085283d1 # v1.38.1
lint-go:
needs: generate
name: GolangCI Lint (${{ matrix.runs-on }} | ${{ matrix.working-directory || 'main' }})
strategy:
fail-fast: false # Get as much feedback as possible in one go...
matrix:
runs-on: [macos-latest, ubuntu-latest, windows-latest]
working-directory: ["", samples]
runs-on: ${{ matrix.runs-on }}
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run golangci-lint
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v6
with:
version: v2.5.0
working-directory: ${{ matrix.working-directory }}
##############################################################################
# Verify all GitHub workflows have hash-pinned actions
lint-actions:
runs-on: ubuntu-latest
name: GitHub Workflow Linters
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Ensure SHA pinned actions
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@9e9574ef04ea69da568d6249bd69539ccc704e74 # v3
with:
allowlist: DataDog/dd-trace-go # Trust actions/workflows in the dd-trace-go repository
- name: Verify actions are pinned with ratchet
uses: sethvargo/ratchet@8b4ca256dbed184350608a3023620f267f0a5253 # ratchet:exclude
with:
files: ".github/workflows/*.yml .github/actions/**/action.yml"
- name: Run actionlint
uses: reviewdog/action-actionlint@f00ad0691526c10be4021a91b2510f0a769b14d0 # v1.68.0
with:
reporter: github-check
level: error
fail_level: error
##############################################################################
# Check code formatting
format:
needs: generate
runs-on: ubuntu-latest
name: Format Check
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run make format
run: make format
- name: Check for formatting changes
run: |-
if [ -n "$(git status --porcelain)" ]; then
echo "::error::Files need formatting. Run 'make format' locally and commit the changes."
git status
git diff
exit 1
fi
##############################################################################
# Run all unit tests with coverage enabled
unit-tests:
needs: generate
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: ${{ github.event_name == 'merge_group' }}
matrix:
# Not running unit tests on macOS, because it's UNIX-like and the only
# os-specific code paths are UNIX vs. Windows.
runs-on: [ubuntu-latest, windows-latest]
go-version: [oldstable, stable]
name: Unit tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }})
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup Go
id: setup-go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.mod"
- name: Run unit tests
shell: bash
run: |-
mkdir -p coverage
test_args=("-shuffle=on" "-race")
if [ "${{ runner.os }}" == "Windows" ]; then
test_args+=("-timeout=30m")
fi
if [ "${{ github.event_name }}" != "merge_group" ]; then
test_args+=("-cover" "-covermode=atomic" "-coverpkg=./...,github.com/DataDog/orchestrion/...")
fi
go test "${test_args[@]}" "-coverprofile=${{ github.workspace }}/coverage/unit.out" ./...
go -C samples test "${test_args[@]}" "-coverprofile=${{ github.workspace }}/coverage/samples.out" ./...
- name: Determine simple go version
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
id: go
shell: bash
run: |-
set -euo pipefail
echo "version=$(echo '${{ steps.setup-go.outputs.go-version }}' | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
- name: Upload coverage report
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
with:
name: coverage-unit+go${{ steps.go.outputs.version }}+${{ runner.os }}+${{ runner.arch }}
path: |-
./coverage/unit.out
./coverage/samples.out
##############################################################################
# Run all benchmarks and generate report
benchmark:
needs: generate
runs-on: arm-8core-linux
name: Benchmarks (run ${{ matrix.run_number }})
strategy:
matrix:
# Run benchmarks 6 times on different runners to smooth out compute discrepancies...
run_number: [1, 2, 3, 4, 5, 6]
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Run benchmarks
run: |-
set -euo pipefail
go test -bench=. -timeout=1h -run=^$ . | tee ${{ runner.temp }}/benchmarks-${{ matrix.run_number }}.txt
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Upload benchmark report (raw)
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
with:
if-no-files-found: error
name: benchmarks-${{ matrix.run_number }}.txt
path: ${{ runner.temp }}/benchmarks-${{ matrix.run_number }}.txt
benchmark-report:
needs: benchmark
runs-on: ubuntu-latest
name: Benchmark Report
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup go
id: setup-go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: stable
cache-dependency-path: "**/go.mod"
- name: Download benchmark reports (raw)
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
with:
pattern: benchmarks-*.txt
path: ${{ runner.temp }}
merge-multiple: true
- name: Format report
run: |-
set -euo pipefail
go -C _tools run golang.org/x/perf/cmd/benchstat \
-table=.name -row=/repo@alpha -col=/variant \
${{ runner.temp }}/benchmarks-*.txt \
| tee ${{ runner.temp }}/benchmarks-formatted.txt
- name: Setting Job Summary
run: |-
{
echo "### Benchmark Report"
echo '```'
cat ${{ runner.temp }}/benchmarks-formatted.txt
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
##############################################################################
# Run all integration tests and gather extensive coverage
integration-tests:
name: Integration Tests
needs: generate
uses: DataDog/dd-trace-go/.github/workflows/orchestrion.yml@399a58b6cb0b5afb9ed6ba6861a3ee8521a11c02 # ratchet:DataDog/dd-trace-go/.github/workflows/orchestrion.yml@main
with:
collect-coverage: ${{ github.event_name != 'merge_group' }}
orchestrion-version: ${{ github.sha }}
secrets:
DD_API_KEY: ${{ secrets.DD_CI_API_KEY }}
##############################################################################
# Run end-to-end tests (PGO, etc.)
e2e-tests:
name: E2E Tests (go ${{ matrix.go-version }}, ${{ matrix.runs-on }})
needs: generate
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-latest, macos-latest, windows-latest]
go-version: [oldstable, stable]
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Setup Go
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.mod"
- name: Build orchestrion
run: go build -o bin/orchestrion .
- name: Run E2E tests
run: go test -tags=e2e -v -timeout=10m ./test/e2e/ 2>&1 | tee test-e2e.log
env:
E2E_TEST_TIMEOUT: 5m
- name: Upload E2E artifacts on failure
if: failure()
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4
with:
name: e2e-artifacts-${{ matrix.runs-on }}-go${{ matrix.go-version }}
path: |
test-e2e.log
test/e2e/**/*.log
test/e2e/**/*.pgo
test/e2e/**/*.prof
if-no-files-found: ignore
##############################################################################
# Assert everything is complete. This simplifies branch protection settings
# and allows us to have one single trigger for CodeCov reporting.
complete:
runs-on: ubuntu-latest
name: Complete
needs:
- generate
- lint
- lint-go
- lint-actions
- format
- unit-tests
- integration-tests
- e2e-tests
- benchmark
if: '!cancelled()'
steps:
- name: Success
if: needs.generate.result != 'failure' && needs.lint.result != 'failure' && needs.lint-go.result != 'failure' && needs.lint-actions.result != 'failure' && needs.format.result != 'failure' && needs.unit-tests.result != 'failure' && needs.integration-tests.result != 'failure' && needs.e2e-tests.result != 'failure'
run: echo "OK"
- name: Failed
if: needs.generate.result == 'failure' || needs.lint.result == 'failure' || needs.lint-go.result == 'failure' || needs.lint-actions.result == 'failure' || needs.format.result == 'failure' || needs.unit-tests.result == 'failure' || needs.integration-tests.result == 'failure' || needs.e2e-tests.result == 'failure'
run: |-
echo "Failed!"
exit 1
##############################################################################
# Produce a CodeCov coverage report with all uploaded code coverage data.
coverage-preflight:
runs-on: ubuntu-latest
name: CodeCov pre-flight
steps:
- name: Checkout
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Download codecov CLI
id: codecov-cli
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
uses: ./.github/actions/codecov-cli
- name: Register commit with CodeCov
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
shell: bash
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: |-
set -euo pipefail
pr=()
sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}"
parentsha="${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}"
if [ "${{ github.event_name }}" == "pull_request" ]; then
pr+=("--pr=${{ github.event.number }}")
fi
echo "::group::Register commit metadata with CodeCov"
${{ steps.codecov-cli.outputs.codecov }} \
--auto-load-params-from=GithubActions \
--verbose \
create-commit \
--parent-sha="${parentsha}" \
${pr[@]+"${pr[@]}"} \
--sha="${sha}" \
--fail-on-error \
--git-service=github \
--token="${CODECOV_TOKEN}" \
--slug="${{ github.repository }}"
echo "::endgroup::"
echo "::group::Create a new blank CodeCov report"
${{ steps.codecov-cli.outputs.codecov }} \
--auto-load-params-from=GithubActions \
--verbose \
create-report \
${pr[@]+"${pr[@]}"} \
--sha="${sha}" \
--fail-on-error \
--git-service=github \
--token="${CODECOV_TOKEN}" \
--slug="${{ github.repository }}"
echo "::endgroup::"
coverage-matrix:
runs-on: ubuntu-latest
name: Compute Coverage Matrix
needs:
- coverage-preflight
- unit-tests
- integration-tests
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
outputs:
artifacts: ${{ steps.compute.outputs.artifacts }}
files: ${{ steps.compute.outputs.files }}
matrix: ${{ steps.compute.outputs.matrix }}
steps:
- name: Setup Node
uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v4
with:
node-version: latest
- name: Download Artifacts
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
with:
pattern: coverage-*
- name: Compute Matrix
id: compute
run: |-
node <<-EOF
const fs = require('node:fs');
const path = require('node:path');
const process = require('node:process');
const flags = [];
const flagFiles = {};
for (const dirname of fs.readdirSync(process.cwd())) {
const prefix = 'coverage-';
if (!dirname.startsWith(prefix)) {
continue;
}
const files = fs.globSync(path.join(process.cwd(), dirname, '**', '*.out'));
console.log('Found asset named ' + dirname + ' with ' + files.length + ' report files.');
if (files.length == 0) {
continue;
}
for (const flag of dirname.substring(prefix.length).split('+')) {
if (!flags.includes(flag)) {
flags.push(flag);
}
flagFiles[flag] ??= [];
flagFiles[flag].push(...files);
}
}
console.log('Flags:', flags);
console.log('Files:', flagFiles);
// Join the lists because the workflow subsequently expects a whitespace-separted list.
for (const [flag, list] of Object.entries(flagFiles)) {
flagFiles[flag] = list.join(' ');
}
fs.writeFileSync(
path.join(process.env.GITHUB_OUTPUT),
[
"matrix=" + JSON.stringify({ flag: flags }),
"files=" + JSON.stringify(flagFiles),
].join('\n'),
);
EOF
coverage-upload:
runs-on: ubuntu-latest
name: Upload report to CodeCov (${{ matrix.flag }})
needs: [coverage-matrix]
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
strategy:
fail-fast: true
matrix: ${{ fromJson(needs.coverage-matrix.outputs.matrix) }}
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Download Artifacts
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v4
with:
pattern: coverage-*
- name: Upload Reports
uses: ./.github/actions/codecov-upload
with:
name: ${{ matrix.flag }}
flags: ${{ matrix.flag }}
files: ${{ fromJson(needs.coverage-matrix.outputs.files)[matrix.flag] }}
token: ${{ secrets.CODECOV_TOKEN }}
coverage-finalize:
runs-on: ubuntu-latest
name: Create CodeCov report
needs: [coverage-upload]
if: github.event_name != 'merge_group' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork)
steps:
- name: Checkout
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v4
- name: Download codecov CLI
id: codecov-cli
uses: ./.github/actions/codecov-cli
- name: Create CodeCov report
shell: bash
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: |-
set -euo pipefail
sha="${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}"
echo "::group::Create CodeCov report results"
${{ steps.codecov-cli.outputs.codecov }} \
--auto-load-params-from=GithubActions \
--verbose \
create-report-results \
--sha="${sha}" \
--fail-on-error \
--git-service=github \
--token="${CODECOV_TOKEN}" \
--slug="${{ github.repository }}"
echo "::endgroup::"
echo "::group::Issue GitHub notifications"
${{ steps.codecov-cli.outputs.codecov }} \
--auto-load-params-from=GithubActions \
--verbose \
send-notifications \
--sha="${sha}" \
--fail-on-error \
--git-service=github \
--token="${CODECOV_TOKEN}" \
--slug=${{ github.repository }}
echo "::endgroup::"