load-tests #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: load-tests | |
| on: | |
| push: | |
| tags: | |
| - "v[0-9]+.[0-9]+.[0-9]+*" | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "27 0,12 * * *" # Run at 00:27 and 12:27 UTC every day. | |
| # Do not cancel this workflow on main. See https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/16616 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| # Make sure to exit early if cache segment download times out after 2 minutes. | |
| # We limit cache download as a whole to 5 minutes. | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2 | |
| permissions: read-all | |
| jobs: | |
| setup-environment: | |
| timeout-minutes: 30 | |
| runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24 | |
| if: ${{ github.actor != 'dependabot[bot]' && github.repository_owner == 'open-telemetry' }} | |
| outputs: | |
| loadtest_matrix: ${{ steps.splitloadtest.outputs.loadtest_matrix }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - uses: ./.github/actions/setup-go-tools | |
| with: | |
| go-version: oldstable | |
| - run: make genoteltestbedcol | |
| - run: make oteltestbedcol | |
| - name: Upload Testbed Binaries | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: testbed-binaries | |
| path: ./bin/* | |
| - name: Split Loadtest Jobs | |
| id: splitloadtest | |
| run: ./.github/workflows/scripts/setup_e2e_tests.sh | |
| - run: ./.github/workflows/scripts/check-disk-space.sh | |
| loadtest: | |
| runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24 | |
| needs: [setup-environment] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.setup-environment.outputs.loadtest_matrix) }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - uses: ./.github/actions/setup-go-tools | |
| with: | |
| go-version: oldstable | |
| - run: mkdir -p results && touch results/TESTRESULTS.md | |
| - name: Download Testbed Binaries | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: testbed-binaries | |
| path: bin/ | |
| - run: chmod +x bin/* | |
| # Remove any pre-installed kwokctl so the action installs the pinned version | |
| - name: Remove existing KWOK binaries | |
| run: | | |
| sudo rm -f /usr/local/bin/kwokctl /usr/local/bin/kwok | |
| - name: Install KWOK (kwokctl) | |
| uses: kubernetes-sigs/kwok@c04392afe3cc6929cd3cc79f61d8bdcd83a8ba5d # v0.7.0 | |
| with: | |
| command: kwokctl | |
| - name: Loadtest | |
| run: make -C testbed run-tests | |
| env: | |
| TEST_ARGS: "-test.run=${{ matrix.test }}" | |
| - name: Set results filename | |
| if: ${{ failure() || success() }} | |
| id: filename | |
| run: echo "name=$(echo '${{ matrix.test }}' | sed -e 's/|/_/g')" >> "$GITHUB_OUTPUT" | |
| - name: Create Test Result Archive | |
| if: ${{ failure() || success() }} | |
| continue-on-error: true | |
| run: tar -cvf test_results_${{steps.filename.outputs.name}}.tar testbed/tests/results | |
| - name: Upload Test Results | |
| if: ${{ failure() || success() }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: test-result-archive-${{steps.filename.outputs.name}} | |
| path: ./*.tar | |
| - run: cp testbed/tests/results/benchmarks.json testbed/tests/results/${{steps.filename.outputs.name}}.json | |
| - name: Upload benchmarks.json | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: benchmark-results-${{steps.filename.outputs.name}} | |
| path: testbed/tests/results/${{steps.filename.outputs.name}}.json | |
| - run: ./.github/workflows/scripts/check-disk-space.sh | |
| update-benchmarks: | |
| runs-on: ubuntu-24.04 | |
| needs: [loadtest] | |
| permissions: | |
| # deployments permission to deploy GitHub pages website | |
| deployments: write | |
| # contents permission to update benchmark contents in gh-pages branch | |
| contents: write | |
| if: github.event_name != 'pull_request' | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: benchmark-results-* | |
| merge-multiple: true | |
| path: results | |
| - run: jq -s 'map(.[])' results/*.json > output.json | |
| - uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7 # v1 | |
| with: | |
| tool: "customSmallerIsBetter" | |
| output-file-path: output.json | |
| gh-pages-branch: benchmarks | |
| max-items-in-chart: 100 | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| benchmark-data-dir-path: "docs/benchmarks/loadtests" | |
| auto-push: true | |
| notify-failure: | |
| runs-on: ubuntu-24.04 | |
| needs: [loadtest] | |
| if: >- | |
| failure() && | |
| github.event_name != 'pull_request' && | |
| github.repository_owner == 'open-telemetry' | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: Download test result archives | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: test-result-archive-* | |
| path: /tmp/test-results/ | |
| continue-on-error: true | |
| - name: Check for existing issue | |
| id: check | |
| run: | | |
| issue_number=$(gh issue list -R ${{ github.repository }} \ | |
| --search "load-tests workflow failed" \ | |
| --state open \ | |
| --label "flaky tests" \ | |
| --json number \ | |
| --jq '.[0].number // empty') | |
| if [ -n "$issue_number" ]; then | |
| echo "exists=true" >> "$GITHUB_OUTPUT" | |
| echo "number=$issue_number" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "exists=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Collect failure details | |
| id: details | |
| run: | | |
| run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| short_sha=$(echo "${{ github.sha }}" | cut -c1-7) | |
| commit_url="${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}" | |
| # Extract PR number from merge commit message (contains "(#NNNNN)") | |
| commit_msg=$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.commit.message' 2>/dev/null || true) | |
| pr_number=$(echo "$commit_msg" | head -1 | grep -oP '\(#\K\d+' | head -1 || true) | |
| # Get failed job names (excluding notify-failure itself) | |
| failed_jobs=$(gh run view "${{ github.run_id }}" -R "${{ github.repository }}" \ | |
| --json jobs \ | |
| -q '[.jobs[] | select(.conclusion == "failure" and (.name | startswith("notify-failure") | not)) | .name] | map("- " + .) | join("\n")') | |
| # Extract performance table and test failures from archived test results | |
| performance_table="" | |
| failure_msgs="" | |
| for tarfile in /tmp/test-results/*/*.tar; do | |
| tar -xf "$tarfile" testbed/tests/results/TESTRESULTS.md 2>/dev/null || true | |
| tar -xf "$tarfile" testbed/tests/results/testoutput.log 2>/dev/null || true | |
| done | |
| if [ -f testbed/tests/results/TESTRESULTS.md ]; then | |
| performance_table=$(cat testbed/tests/results/TESTRESULTS.md) | |
| fi | |
| if [ -f testbed/tests/results/testoutput.log ]; then | |
| failure_msgs=$(grep -E '(--- FAIL|^FAIL)' testbed/tests/results/testoutput.log | head -20) | |
| fi | |
| # Build issue body to a file to avoid shell escaping issues | |
| { | |
| echo "Auto-generated report for load-tests workflow." | |
| echo "" | |
| echo "Link to failed build: ${run_url}" | |
| echo "Commit: [${short_sha}](${commit_url})" | |
| if [ -n "$pr_number" ]; then | |
| echo "PR: [#${pr_number}](${{ github.server_url }}/${{ github.repository }}/pull/${pr_number})" | |
| fi | |
| echo "" | |
| echo "### Component(s)" | |
| echo "" | |
| echo "testbed" | |
| echo "" | |
| echo "### Failed jobs" | |
| echo "" | |
| echo "$failed_jobs" | |
| if [ -n "$performance_table" ]; then | |
| echo "" | |
| echo "### Performance results" | |
| echo "" | |
| echo '```' | |
| echo "$performance_table" | |
| echo '```' | |
| fi | |
| if [ -n "$failure_msgs" ]; then | |
| echo "" | |
| echo "#### Test failures" | |
| echo "" | |
| echo '```' | |
| echo "$failure_msgs" | |
| echo '```' | |
| fi | |
| echo "" | |
| echo "**Note**: Information about any subsequent build failures that happen while" | |
| echo "this issue is open, will be added as comments with more information to this issue." | |
| } > /tmp/issue-body.md | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Create issue | |
| if: steps.check.outputs.exists == 'false' | |
| run: | | |
| gh issue create -R "${{ github.repository }}" \ | |
| -t 'load-tests workflow failed' \ | |
| --body-file /tmp/issue-body.md \ | |
| -l 'needs triage' \ | |
| -l 'flaky tests' \ | |
| -l 'ci-cd' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Comment on existing issue | |
| if: steps.check.outputs.exists == 'true' | |
| run: | | |
| gh issue comment "${{ steps.check.outputs.number }}" \ | |
| -R "${{ github.repository }}" \ | |
| --body-file /tmp/issue-body.md | |
| env: | |
| GH_TOKEN: ${{ github.token }} |