Smoke Tests #14671

Workflow file for this run

.github/workflows/smoke-tests.yml at 85a3c43

	name: Smoke Tests

	on:
	workflow_call: # allows to reuse this workflow
	inputs:
	ref:
	description: The branch or tag to run the workflow on
	required: true
	type: string
	go-libddwaf-ref:
	description: A git ref to update github.com/DataDog/go-libddwaf/v5 to. No-op if empty.
	required: false
	type: string
	push:
	branches:
	- release-v*
	- main
	tags-ignore:
	- 'contrib/**'
	- 'instrumentation/**'
	- 'internal/**'
	- 'orchestrion/**'
	- 'scripts/**'
	schedule: # nightly
	- cron: "0 0 * * *"
	workflow_dispatch: { } # manually

	env:
	TEST_RESULTS: /tmp/test-results # path to where test results will be saved
	GOPROXY: "proxy.golang.org\|direct" # fall back to `direct` in case of proxy issues

	permissions:
	id-token: write
	contents: read

	jobs:
	setup-env:
	runs-on: ubuntu-latest
	timeout-minutes: 20 # normal runtime is ~5m; bounds a toolchain-corruption retry loop
	outputs:
	matrix: ${{ steps.matrix.outputs.matrix }}
	env:
	PACKAGES: ./internal/... ./ddtrace/... ./profiler/... ./appsec/... ./instrumentation/...
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ inputs.ref \|\| github.ref }}
	repository: DataDog/dd-trace-go

	- name: Setup Go and development tools
	uses: ./.github/actions/setup-go
	with:
	# Pinned off `stable` (currently 1.26.5), which intermittently corrupts
	# its own heap mid-build (golang/go#77168) and fails these sandboxed
	# smoke jobs. Restore `stable` once a fixed 1.26.x ships.
	go-version: "1.26.4"
	tools-dir: ${{ github.workspace }}/_tools
	tools-bin: ${{ github.workspace }}/bin

	- name: Compile dd-trace-go with updated dependencies (sandboxed)
	# go get -u executes dependency code from the network. Keep those
	# changes inside the gVisor sandbox; this setup job only needs to prove
	# the updated dependency graph still compiles.
	uses: geomys/sandboxed-step@a6bfc174e532cea4f378e3427bcde8b714c2e810 # v1.2.2
	with:
	env: \|
	GOPROXY=${{ env.GOPROXY }}
	GO_LIBDDWAF_REF=${{ inputs.go-libddwaf-ref }}
	PACKAGES=${{ env.PACKAGES }}
	TEST_RESULTS=${{ env.TEST_RESULTS }}
	run: \|-
	mkdir -p "$TEST_RESULTS"
	compile() {
	# `until` conditions (see retry_on_corruption in go-retry.sh) suspend `-e` for
	# the commands they wrap, including inside this function's body — a failed step
	# here would otherwise go unnoticed as long as the final `go build` happens to
	# still succeed. Propagate each failure explicitly.
	# shellcheck disable=SC2086
	go get -u -t $PACKAGES \|\| return
	go mod tidy \|\| return
	find . -iname go.mod -exec dirname {} \; \| while read -r d; do pushd "$d" && go mod tidy && popd \|\| exit 1; done \|\| return
	if [ -n "${GO_LIBDDWAF_REF:-}" ]; then
	go get -u -t "github.com/DataDog/go-libddwaf/v5@${GO_LIBDDWAF_REF}" \|\| return
	go mod tidy \|\| return
	fi
	# shellcheck disable=SC2086
	go build $PACKAGES
	}
	# See go-retry.sh for why toolchain/cache corruption is retried instead of failing
	# the job outright, and why a failure with none of the known signatures is treated
	# as a genuine break and fails immediately, without retry.
	source ./.github/workflows/apps/go-retry.sh
	retry_on_corruption compile

	- name: Compute Matrix
	id: matrix
	run: \|-
	echo -n "matrix=" >> "${GITHUB_OUTPUT}"
	go run ./scripts/ci_contrib_matrix.go >> "${GITHUB_OUTPUT}"

	go-get-u:
	# Run go get -u to upgrade dd-trace-go dependencies to their
	# latest minor version and see if dd-trace-go still compiles.
	# Related to issue https://github.com/DataDog/dd-trace-go/issues/1607
	name: 'go get -u smoke test'
	needs: setup-env
	timeout-minutes: 45 # normal chunk runtime is ~10-25m; bounds a toolchain-corruption retry loop
	strategy:
	fail-fast: false
	matrix:
	chunk: ${{ fromJson(needs.setup-env.outputs.matrix) }}
	runs-on: ubuntu-latest
	if: github.repository_owner == 'DataDog' # only run on DataDog's repository, not in forks
	env:
	TEST_RESULTS: ${{ github.workspace }}/test-results
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ inputs.ref \|\| github.ref }}
	# Manually specify the repository, which is necessary in the workflow_call situation, as the default is
	# otherwise the repository where the caller workflow started from. In this case, we need to check out the
	# repository where the called workflow is (i.e, this repository); but I don't know of a more elegant way to
	# obtain its name than hard-coding it.
	repository: DataDog/dd-trace-go
	- name: Setup Go and development tools
	uses: ./.github/actions/setup-go
	with:
	# Pinned off `stable` (currently 1.26.5), which intermittently corrupts
	# its own heap mid-build (golang/go#77168) and fails this sandboxed
	# go-get-u smoke job. Restore `stable` once a fixed 1.26.x ships.
	go-version: "1.26.4"
	tools-dir: ${{ github.workspace }}/_tools
	tools-bin: ${{ github.workspace }}/bin
	- name: Capture trusted commit SHA
	# Pinned before any sandboxed step so we can re-checkout this exact
	# commit after the sandbox runs. Anything that escapes the sandbox
	# could otherwise rewrite .git/HEAD or refs and trick the host into
	# running attacker-controlled code via local actions.
	id: trusted
	run: echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
	- name: Test contribs (sandboxed)
	# It needs to run before "Test dd-trace-go" to avoid TestTelemetryEnabled tests to fail.
	# The smoke script runs go get -u inside each contrib module, so run it
	# in gVisor and persist only the generated reports/coverage for uploads.
	uses: geomys/sandboxed-step@a6bfc174e532cea4f378e3427bcde8b714c2e810 # v1.2.2
	with:
	persist-workspace-changes: 'true'
	env: \|
	GOPROXY=${{ env.GOPROXY }}
	TEST_RESULTS=${{ env.TEST_RESULTS }}
	run: \|-
	export PATH="${GITHUB_WORKSPACE}/bin:${PATH}"
	# ci_test_contrib.sh already retries each individual `go get`/`go mod tidy` call via
	# go-retry.sh (see that file for why toolchain/cache corruption is retried while
	# genuine breaks fail immediately). Don't wrap this whole multi-contrib loop in a
	# second, outer retry_on_corruption: that would re-run every contrib in the chunk,
	# including ones that already succeeded, on top of the inner retries — multiplying
	# both work and cache-wipe churn instead of just retrying the one command that failed.
	./scripts/ci_test_contrib.sh smoke ${{ toJson(matrix.chunk) }}
	- name: Stage sandbox artifacts outside workspace
	# Move the test results and coverage files to RUNNER_TEMP so the
	# next checkout (which wipes the workspace) does not delete them.
	# Only regular files are staged, to avoid following any symlinks
	# that may have been planted inside the sandbox.
	id: stage
	if: always()
	run: \|-
	set -euo pipefail
	SAFE="${RUNNER_TEMP}/sandbox-artifacts"
	rm -rf "${SAFE}"
	mkdir -p "${SAFE}"
	if [ -d "${TEST_RESULTS}" ]; then
	( cd "${TEST_RESULTS}" && find . -type f -print0 \
	\| tar --null --no-recursion -T - -cf "${SAFE}/test-results.tar" )
	fi
	( cd "${GITHUB_WORKSPACE}" \
	&& find . -type f $ -name 'coverage*.txt' -o -name 'profile.cov' $ -print0 \
	\| tar --null --no-recursion -T - -cf "${SAFE}/coverage.tar" ) \|\| true
	- name: Re-checkout trusted commit after sandbox
	# Wipe the (potentially tampered) workspace and check out the SHA we
	# captured before the sandbox ran. This restores .git, tracked files,
	# and any local actions to a known-good state before any credentialed
	# local action is invoked.
	id: recheckout
	if: always()
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ steps.trusted.outputs.sha }}
	repository: DataDog/dd-trace-go
	clean: true
	- name: Restore sandbox artifacts
	id: restore
	if: always() && steps.recheckout.outcome == 'success'
	run: \|-
	set -euo pipefail
	SAFE="${RUNNER_TEMP}/sandbox-artifacts"
	mkdir -p "${TEST_RESULTS}"
	if [ -f "${SAFE}/test-results.tar" ]; then
	tar -C "${TEST_RESULTS}" -xf "${SAFE}/test-results.tar"
	fi
	if [ -f "${SAFE}/coverage.tar" ]; then
	tar -C "${GITHUB_WORKSPACE}" -xf "${SAFE}/coverage.tar"
	fi
	- name: Get Datadog credentials
	id: dd-sts
	if: always() && steps.restore.outcome == 'success'
	continue-on-error: true
	uses: DataDog/dd-sts-action@639d841c72f15e4e77747bd726ef8105ce971da2
	with:
	policy: dd-trace-go
	- name: Upload the results to Datadog CI App
	if: always() && steps.restore.outcome == 'success'
	continue-on-error: true
	uses: ./.github/actions/dd-ci-upload
	with:
	dd-api-key: ${{ steps.dd-sts.outputs.api_key }}
	path: ${{ env.TEST_RESULTS }}
	tags: go:stable,arch:${{ runner.arch }},os:${{ runner.os }}

	- name: Merge contrib coverage files
	if: always() && steps.restore.outcome == 'success'
	run: \|
	echo "mode: atomic" > coverage-contrib-merged.txt
	find . $ -path "/contrib/" -o -path "/instrumentation/" $ \
	-name "coverage-*.txt" -print0 \
	\| xargs -0 grep -h -v "^mode:" >> coverage-contrib-merged.txt 2>/dev/null \|\| true

	- name: Upload Coverage to Datadog
	if: always() && steps.restore.outcome == 'success'
	continue-on-error: true
	uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77
	with:
	api_key: ${{ steps.dd-sts.outputs.api_key }}
	files: coverage-contrib-merged.txt
	format: go-coverprofile

	smoke-test-tracer:
	name: Test dd-trace-go
	runs-on: ubuntu-latest
	env:
	PACKAGES: ./internal/... ./ddtrace/... ./profiler/... ./appsec/... ./instrumentation/...
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ inputs.ref \|\| github.ref }}
	repository: DataDog/dd-trace-go
	- name: Setup Go and development tools
	uses: ./.github/actions/setup-go
	with:
	go-version: "1.25"
	tools-dir: ${{ github.workspace }}/_tools
	tools-bin: ${{ github.workspace }}/bin
	- name: Run tests
	run: \|
	export PATH="${{ github.workspace }}/bin:${PATH}"
	# shellcheck disable=SC2086
	gotestsum --junitfile "${TEST_RESULTS}/gotestsum-report-smoke-tracer.xml" -- -v $PACKAGES
	# Check for changes in the supported_configurations.json file
	- name: Supported Configurations Diff Check
	if: always()
	uses: ./.github/actions/supported_configurations_validation
	- name: Get Datadog credentials
	id: dd-sts
	if: always()
	continue-on-error: true
	uses: DataDog/dd-sts-action@639d841c72f15e4e77747bd726ef8105ce971da2
	with:
	policy: dd-trace-go
	- name: Upload the results to Datadog CI App
	if: always()
	continue-on-error: true
	uses: ./.github/actions/dd-ci-upload
	with:
	dd-api-key: ${{ steps.dd-sts.outputs.api_key }}
	path: ${{ env.TEST_RESULTS }}
	tags: go:1.25,arch:${{ runner.arch }},os:${{ runner.os }}

	check-gen-files:
	name: Check generated files and go mod files are up-to-date.
	runs-on: ubuntu-latest
	if: inputs.go-libddwaf-ref == ''
	env:
	# Users may build our library with GOTOOLCHAIN=local. If they do, and our
	# go.mod file specifies a newer Go version than their local toolchain, their
	# build will break. Run our tests with GOTOOLCHAIN=local to ensure that
	# our library builds with all of the Go versions we claim to support,
	# without having to download a newer one.
	GOTOOLCHAIN: local
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ inputs.ref \|\| github.ref }}
	- name: Setup Go and development tools
	uses: ./.github/actions/setup-go
	with:
	go-version: "1.25"
	tools-dir: ${{ github.workspace }}/_tools
	tools-bin: ${{ github.workspace }}/bin
	- name: Run generate scripts and check diff
	run: \|-
	export PATH="${{ github.workspace }}/bin:${PATH}"
	./scripts/generate.sh
	./scripts/fix_modules.sh
	git diff --exit-code

	# TODO: macos setup requirements (xcode tools installation, etc.)
	setup-requirements-linux:
	# Build and deployment setup smoke test of linux containers built from the
	# golang docker image to test that dd-trace-go doesn't need more than the
	# "out-of-the-box" images. It is expected require a few more tools when CGO
	# is enabled, but nothing more than gcc and the C library, but nothing more.
	# Anything more than this "standard Go build and deployment requirements"
	# must be considered breaking changes.
	name: 'Build and deployment requirements smoke tests'
	runs-on: ubuntu-latest
	if: github.repository_owner == 'DataDog' # only run on DataDog's repository, not in forks
	strategy:
	fail-fast: false
	matrix:
	# TODO: cross-compilation from/to different hardware architectures once
	# github provides native ARM runners.
	go: [ "1.26", "1.25" ]
	build-env: [ alpine, trixie, bookworm ]
	build-with-cgo: [ 0, 1 ]
	deployment-env: [ alpine, debian12, debian13, al2, al2023, busybox, scratchy ]
	include:
	# GitHub limits the number of matrix jobs to 256, so we need to reduce
	# it a bit, and we can reduce redundant tests.
	# 1. Building with `go mod vendoring` is not worth it on all the
	# possible build and deployment envs.
	- build-env: alpine
	build-with-vendoring: y
	build-with-cgo: 1 # cgo's build tag can impact the vendored files
	deployment-env: alpine
	- build-env: alpine
	build-with-vendoring: y
	build-with-cgo: 0 # cgo's build tag can impact the vendored files
	deployment-env: alpine
	# 2. Given the low blast radius of the busybox deployment environment
	# this is the only one where we accept the libdl.so.2 requirement.
	# For this reason, we add the datadog.no_waf build tag in this only
	# case to avoid the libdl.so.2 dependency.
	- deployment-env: busybox
	build-with-cgo: 1
	build-tags: "datadog.no_waf"

	exclude:
	# Exclude "out of the box" cases requiring extra setup:
	# 1. Building with CGO enabled on alpine but deploying to a non-alpine
	# environment: the C library isn't located at the same place.
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: debian12
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: debian13
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: al2
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: al2023
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: busybox
	- build-env: alpine
	build-with-cgo: 1
	deployment-env: scratchy
	# 2. Too old glibc on the deployment environment than on the build env
	- build-env: bookworm
	deployment-env: al2
	- build-env: trixie
	deployment-env: al2
	# 3. Build with CGO enabled and deploying to a scratch/busybox docker
	# image requires copying the dynamic lib dependencies (full example
	# provided at https://github.com/DataDog/appsec-go-test-app/blob/main/examples/docker/scratch/Dockerfile)
	- build-with-cgo: 1
	deployment-env: scratchy

	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	ref: ${{ inputs.ref \|\| github.ref }}
	# Manually specify the repository, which is necessary in the workflow_call situation, as the default is
	# otherwise the repository where the caller workflow started from. In this case, we need to check out the
	# repository where the called workflow is (i.e, this repository); but I don't know of a more elegant way to
	# obtain its name than hard-coding it.
	repository: DataDog/dd-trace-go
	- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
	- name: Build
	uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
	with:
	context: .
	file: ./internal/setup-smoke-test/Dockerfile
	push: false
	load: true
	tags: smoke-test
	build-args: \|
	go=${{ matrix.go }}
	build_env=${{ matrix.build-env }}
	build_tags=${{ matrix.build-tags }}
	build_with_vendoring=${{ matrix.build-with-vendoring }}
	build_with_cgo=${{ matrix.build-with-cgo }}
	deployment_env=${{ matrix.deployment-env }}
	go_libddwaf_ref=${{ inputs.go-libddwaf-ref }}
	- name: Test
	run: docker run -p7777:7777 --rm smoke-test

	check-orchestrion-all-resolves:
	# Verify that orchestrion/all/v2, and every dd-trace-go module it requires, can be
	# resolved from the module proxy by an external consumer.
	name: 'Check orchestrion/all resolves from the module proxy'
	runs-on: ubuntu-latest
	if: github.repository_owner == 'DataDog' # needs a commit pushed to the public repo
	env:
	REF: ${{ inputs.ref \|\| github.sha }}
	# This job checks that modules can be downloaded, not that their checksums match,
	# which the other jobs already cover. Skipping the checksum database avoids a false
	# failure when it has not yet caught up with a just-pushed commit.
	GOSUMDB: "off"
	steps:
	- name: Setup Go
	uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
	with:
	go-version: "1.25" # lowest supported version, so this checks the floor
	- name: Resolve orchestrion/all as an external consumer would
	id: resolve
	run: \|-
	set -euo pipefail
	module=github.com/DataDog/dd-trace-go/orchestrion/all/v2

	# Build the throwaway consumer outside the workspace, so no go.work or replace
	# directive above it can influence resolution.
	work=$(mktemp -d)
	cd "$work"
	go mod init ci.local/orchestrion-all-resolve-check

	# orchestrion/all only pulls the contrib modules in under the `tools` build
	# tag, which is how `orchestrion pin` wires it into a user's project. A plain
	# `go get` prunes them away and passes even when a tag is missing, so the tool
	# file plus `go mod tidy`, which considers every build tag, is what actually
	# forces each contrib module to resolve.
	printf '//go:build tools\n\npackage tools\n\nimport _ "%s"\n' "$module" > orchestrion.tool.go

	echo "::group::go get $module@$REF"
	go get "$module@$REF"
	echo "::endgroup::"

	echo "::group::go mod tidy"
	go mod tidy
	echo "::endgroup::"
	- name: Explain how to fix this
	if: failure() && steps.resolve.outcome == 'failure'
	run: \|-
	cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
	## `orchestrion/all` is not resolvable from the module proxy

	A new nested module was most likely added without a published tag, so external
	consumers cannot use `orchestrion/all/v2@main` even though this repository builds
	fine. Please run the following to cut a new dev release:

	```sh
	git checkout -b dev-vX.Y.x origin/main
	go run ./scripts/autoreleasetagger --version vX.Y.0-dev.N --root .
	```
	EOF

	smoke-tests-done:
	name: Smoke Tests
	needs:
	- setup-requirements-linux
	- smoke-test-tracer
	- go-get-u
	- check-orchestrion-all-resolves
	runs-on: ubuntu-latest
	if: success() \|\| failure()
	steps:
	- name: Success
	if: needs.setup-requirements-linux.result == 'success' && needs.smoke-test-tracer.result == 'success' && needs.go-get-u.result == 'success' && needs.check-orchestrion-all-resolves.result == 'success'
	run: echo "Success!"
	- name: Failure
	if: needs.setup-requirements-linux.result != 'success' \|\| needs.smoke-test-tracer.result != 'success' \|\| needs.go-get-u.result != 'success' \|\| needs.check-orchestrion-all-resolves.result != 'success'
	uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
	with:
	webhook-type: incoming-webhook
	payload: \|
	{
	"text": "Smoke Tests Failed",
	"linux_result": "${{ needs.setup-requirements-linux.result }}",
	"core_result": "${{ needs.smoke-test-tracer.result }}",
	"contrib_result": "${{ needs.go-get-u.result }}",
	"proxy_resolution_result": "${{ needs.check-orchestrion-all-resolves.result }}",
	"blocks": [
	{
	"type": "header",
	"text": {
	"type": "plain_text",
	"text": ":x: Smoke Tests Failed on Main!"
	}
	},
	{
	"type": "section",
	"fields": [
	{
	"type": "mrkdwn",
	"text": "Repository:\n${{ github.repository }}"
	},
	{
	"type": "mrkdwn",
	"text": "Branch:\n${{ github.ref_name }}"
	},
	{
	"type": "mrkdwn",
	"text": "Workflow:\n${{ github.workflow }}"
	},
	{
	"type": "mrkdwn",
	"text": "Linux Result:\n${{ needs.setup-requirements-linux.result }}"
	},
	{
	"type": "mrkdwn",
	"text": "Tracer Result:\n${{ needs.smoke-test-tracer.result }}"
	},
	{
	"type": "mrkdwn",
	"text": "Contrib Test Result:\n${{ needs.go-get-u.result }}"
	},
	{
	"type": "mrkdwn",
	"text": "Proxy Resolution Result:\n${{ needs.check-orchestrion-all-resolves.result }}"
	}
	]
	}
	]
	}
	env:
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SMOKE_TESTS_WEBHOOK_URL }}

Navigation Menu

Uh oh!

Smoke Tests #14671

Workflow file

Smoke Tests #14671

Uh oh!

Workflow file for this run