CI Testing Upgrade #20279
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build & Test Debug | |
| env: | |
| INTEGRATION_TEST_SHARD_COUNT: 6 | |
| # Cancel older jobs on PRs when new changes are pushed | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| on: | |
| push: | |
| branches: [ master, staging, stable, Starlight, starlight-dev, upstream-devtest, fix/tests ] | |
| merge_group: | |
| pull_request: | |
| types: [ opened, reopened, synchronize, ready_for_review ] | |
| branches: [ master, staging, stable, Starlight, starlight-dev, upstream-devtest, main ] | |
| workflow_dispatch: | |
| workflow_call: | |
| inputs: | |
| harvest: | |
| description: "Run as a timings harvest: skip retries and publish the shard manifest." | |
| type: boolean | |
| default: false | |
| # Fork pull requests run this workflow with no repository token privileges beyond | |
| # read-only source access. PR comments are handled by test-report-comment.yml. | |
| permissions: {} | |
| jobs: | |
| build: | |
| if: github.actor != 'PJBot' && github.event.pull_request.draft == false | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| integration_test_filters: ${{ steps.integration-test-shards.outputs.filters }} | |
| integration_test_shards: ${{ steps.integration-test-shards.outputs.shards }} | |
| steps: | |
| - name: Checkout Master | |
| uses: actions/checkout@v7 | |
| - name: Setup Submodule | |
| run: | | |
| git submodule update --init --recursive | |
| - name: Pull engine updates | |
| uses: space-wizards/submodule-dependency@v0.1.5 | |
| - name: Update Engine Submodules | |
| run: | | |
| cd RobustToolbox/ | |
| git submodule update --init --recursive | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v6 | |
| - name: Build Project | |
| run: dotnet build --configuration DebugOpt /m | |
| - name: Discover Integration Tests | |
| id: integration-test-shards | |
| run: | | |
| bin/Content.IntegrationTests/Content.IntegrationTests --list-tests json | \ | |
| python3 Tools/_Starlight/partition_tests.py generate "$INTEGRATION_TEST_SHARD_COUNT" .integration-filters | |
| echo "filters=$(jq -c '.filters' .integration-filters/manifest.json)" >> "$GITHUB_OUTPUT" | |
| echo "shards=$(jq -c '.filters | keys' .integration-filters/manifest.json)" >> "$GITHUB_OUTPUT" | |
| - name: Upload shard manifest | |
| if: inputs.harvest | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: shard-manifest | |
| path: .integration-filters/manifest.json | |
| retention-days: 1 | |
| - name: Archive test binaries | |
| run: tar --use-compress-program='zstd -T0' -cf /tmp/test-binaries.tar.zst bin | |
| - name: Upload test binaries | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test-binaries | |
| path: /tmp/test-binaries.tar.zst | |
| retention-days: 1 | |
| compression-level: 0 | |
| content-tests: | |
| needs: build | |
| name: Content Tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v6 | |
| - name: Download test binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: test-binaries | |
| - name: Extract test binaries | |
| run: tar --use-compress-program='zstd -d -T0' -xf test-binaries.tar.zst && rm test-binaries.tar.zst | |
| - name: Run Content.Tests | |
| env: | |
| HARVEST: ${{ inputs.harvest }} | |
| RUN_DEBUG: ${{ runner.debug }} | |
| run: | | |
| PROGRESS=() | |
| if [[ "$HARVEST" = "true" || "$RUN_DEBUG" = "1" ]]; then | |
| PROGRESS=(--progress on --output Detailed ) | |
| fi | |
| VERBOSITY=() | |
| if [ "$RUN_DEBUG" = "1" ]; then | |
| VERBOSITY=(--show-stdout All --show-stderr All) | |
| fi | |
| bin/Content.Tests/Content.Tests \ | |
| --retry-failed-tests 2 \ | |
| "${PROGRESS[@]}" \ | |
| "${VERBOSITY[@]}" \ | |
| --report-ctrf \ | |
| --report-ctrf-filename ctrf/ctrf-report.json \ | |
| --results-directory /tmp/test-results \ | |
| --settings .github/ci.runsettings | |
| - name: Upload Content.Tests results | |
| uses: actions/upload-artifact@v6 | |
| if: always() | |
| with: | |
| name: test-nunit-content | |
| path: | | |
| /tmp/test-results/ctrf/ctrf-report.json | |
| /tmp/test-results/Retries/**/ctrf/ctrf-report.json | |
| retention-days: 7 | |
| if-no-files-found: error | |
| integration-tests: | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJSON(needs.build.outputs.integration_test_shards) }} | |
| name: "Integration Tests (shard ${{ matrix.shard }})" | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Submodule | |
| run: | | |
| git submodule update --init --recursive | |
| - name: Pull engine updates | |
| uses: space-wizards/submodule-dependency@v0.1.5 | |
| - name: Update Engine Submodules | |
| run: | | |
| cd RobustToolbox/ | |
| git submodule update --init --recursive | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v6 | |
| - name: Download test binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: test-binaries | |
| - name: Extract test binaries | |
| run: tar --use-compress-program='zstd -d -T0' -xf test-binaries.tar.zst && rm test-binaries.tar.zst | |
| - name: Run Content.IntegrationTests | |
| env: | |
| HARVEST: ${{ inputs.harvest }} | |
| RUN_DEBUG: ${{ runner.debug }} | |
| TEST_FILTER: ${{ fromJSON(needs.build.outputs.integration_test_filters)[matrix.shard] }} | |
| run: | | |
| if [ -z "$TEST_FILTER" ]; then | |
| echo "No tests assigned to shard ${{ matrix.shard }}" | |
| exit 0 | |
| fi | |
| PROGRESS=() | |
| if [[ "$HARVEST" = "true" || $RUN_DEBUG == 1 ]]; then | |
| PROGRESS=(--progress on --output Detailed) | |
| while :; do | |
| free --human --line --seconds 1 --count 1 | tr -d '\r\n' | |
| echo | |
| sleep 10 | |
| done & | |
| MEMWATCH=$! | |
| trap 'kill "$MEMWATCH" 2>/dev/null || true' EXIT | |
| fi | |
| RETRY=(--retry-failed-tests 2) | |
| if [ "$HARVEST" = "true" ]; then | |
| RETRY=() | |
| fi | |
| VERBOSITY=() | |
| if [ "$RUN_DEBUG" = "1" ]; then | |
| VERBOSITY=(--show-stdout All --show-stderr All) | |
| fi | |
| bin/Content.IntegrationTests/Content.IntegrationTests \ | |
| --filter "$TEST_FILTER" \ | |
| --minimum-expected-tests 1 \ | |
| "${RETRY[@]}" \ | |
| "${PROGRESS[@]}" \ | |
| "${VERBOSITY[@]}" \ | |
| --hangdump \ | |
| --hangdump-timeout 3m \ | |
| --hangdump-filename "hang-dump-shard-${{ matrix.shard }}-{pname}-{pid}-{time}.dmp" \ | |
| --report-ctrf \ | |
| --report-ctrf-filename "shard-${{ matrix.shard }}-ctrf-report-{time}.json" \ | |
| --results-directory /tmp/test-results/ \ | |
| --settings .github/ci.runsettings | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v6 | |
| if: always() | |
| with: | |
| name: test-nunit-shard-${{ matrix.shard }} | |
| path: | | |
| /tmp/test-results/shard-*-ctrf-report-*.json | |
| /tmp/test-results/Retries/**/shard-*-ctrf-report-*.json | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload hang dump | |
| id: upload-hang-dump | |
| uses: actions/upload-artifact@v6 | |
| if: always() | |
| with: | |
| name: hang-dump-shard-${{ matrix.shard }} | |
| path: | | |
| /tmp/test-results/**/hang-dump-shard-*.dmp | |
| /tmp/test-results/**/hang-dump-shard-*.log | |
| retention-days: 3 | |
| if-no-files-found: ignore | |
| - name: Report hung tests | |
| if: always() | |
| continue-on-error: true | |
| env: | |
| HANG_DUMP_ARTIFACT_URL: ${{ steps.upload-hang-dump.outputs.artifact-url }} | |
| run: | | |
| shopt -s nullglob globstar | |
| dumps=(/tmp/test-results/**/hang-dump-shard-*.dmp) | |
| logs=(/tmp/test-results/**/hang-dump-shard-*.log) | |
| if [ ${#dumps[@]} -eq 0 ] && [ ${#logs[@]} -eq 0 ]; then | |
| exit 0 | |
| fi | |
| echo "## :warning: Hung tests detected (shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY" | |
| if [ ${#logs[@]} -gt 0 ]; then | |
| echo '```text' >> "$GITHUB_STEP_SUMMARY" | |
| for log in "${logs[@]}"; do | |
| cat "$log" >> "$GITHUB_STEP_SUMMARY" | |
| done | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| if [ -n "$HANG_DUMP_ARTIFACT_URL" ]; then | |
| echo ":floppy_disk: [Download hang dump]($HANG_DUMP_ARTIFACT_URL)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| yaml-linter: | |
| name: YAML Linter | |
| needs: build | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Submodule | |
| run: | | |
| git submodule update --init --recursive | |
| - name: Pull engine updates | |
| uses: space-wizards/submodule-dependency@v0.1.5 | |
| - name: Update Engine Submodules | |
| run: | | |
| cd RobustToolbox/ | |
| git submodule update --init --recursive | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v6 | |
| - name: Download test binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: test-binaries | |
| - name: Extract test binaries | |
| run: tar --use-compress-program='zstd -d -T0' -xf test-binaries.tar.zst && rm test-binaries.tar.zst | |
| - name: Run Linter | |
| run: bin/Content.YAMLLinter/Content.YAMLLinter | |
| ci-success: | |
| name: Debug CI Required | |
| if: ${{ always() }} | |
| needs: | |
| - build | |
| - content-tests | |
| - integration-tests | |
| - yaml-linter | |
| runs-on: ubuntu-slim | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Download CTRF test results | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: test-nunit-* | |
| path: /tmp/ctrf-results | |
| - name: Generate Content Tests report | |
| if: always() | |
| uses: ctrf-io/github-test-reporter@v1 | |
| with: | |
| title: Content Test Results | |
| report-path: '/tmp/ctrf-results/test-nunit-content/**/*.json' | |
| artifact-name: ct-report | |
| upload-artifact: true | |
| github-report: true | |
| failed-folded-report: true | |
| - name: Generate Integration Tests report | |
| if: always() | |
| uses: ctrf-io/github-test-reporter@v1 | |
| with: | |
| report-path: '/tmp/ctrf-results/test-nunit-shard-*/**/*.json' | |
| title: Integration Test Results | |
| github-report: true | |
| failed-folded-report: true | |
| artifact-name: it-report | |
| upload-artifact: true | |
| - name: Fail if any dependency was not successful | |
| if: ${{ needs.build.result != 'success' || needs['content-tests'].result != 'success' || needs['integration-tests'].result != 'success' || needs['yaml-linter'].result != 'success' }} | |
| run: | | |
| echo "build=${{ needs.build.result }}" | |
| echo "content-tests=${{ needs['content-tests'].result }}" | |
| echo "integration-tests=${{ needs['integration-tests'].result }}" | |
| echo "yaml-linter=${{ needs['yaml-linter'].result }}" | |
| exit 1 | |
| - name: CI succeeded | |
| run: exit 0 | |
| cleanup: | |
| name: Cleanup Artifacts | |
| if: always() | |
| needs: | |
| - content-tests | |
| - integration-tests | |
| - yaml-linter | |
| - ci-success | |
| runs-on: ubuntu-slim | |
| steps: | |
| - name: Delete test binaries | |
| if: ${{ !(runner.debug == '1' || inputs.harvest == true) }} | |
| continue-on-error: true | |
| uses: geekyeggo/delete-artifact@v6 | |
| with: | |
| name: | | |
| test-* |