Find mutants #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Find mutants | |
| on: | |
| schedule: | |
| - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| SHARDS: 64 | |
| RUST_BACKTRACE: 1 | |
| MUTANTS_ARGS: --no-shuffle --in-place --profile mutants -- --all-targets -- -Zunstable-options --fail-fast | |
| jobs: | |
| baseline: | |
| name: Baseline test | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| timeout: ${{ steps.baseline.outputs.timeout }} | |
| shards: ${{ steps.baseline.outputs.shards }} | |
| max_shard: ${{ steps.baseline.outputs.max_shard }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| - id: nss-version | |
| run: echo "minimum=$(cat min_version.txt)" >> "$GITHUB_OUTPUT" | |
| - uses: ./.github/actions/nss | |
| with: | |
| minimum-version: ${{ steps.nss-version.outputs.minimum }} | |
| - uses: ./.github/actions/rust | |
| with: | |
| version: nightly | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run baseline test | |
| id: baseline | |
| run: | | |
| SECONDS=0 | |
| cargo test --all-targets -- -Zunstable-options --fail-fast | |
| { | |
| # Minimum timeout is 30s, maximum is 90s, otherwise 3x the baseline test time. | |
| echo "timeout=$(( SECONDS * 3 < 30 ? 30 : SECONDS * 3 > 90 ? 90 : SECONDS * 3 ))" | |
| echo "shards=$(jq -nc '[$ARGS.positional[] | tonumber]' --args $(seq 0 $((SHARDS - 1))))" | |
| echo "max_shard=$((SHARDS - 1))" | |
| } >> "$GITHUB_OUTPUT" | |
| mutants: | |
| name: Find mutants (shard ${{ matrix.shard }}/${{ needs.baseline.outputs.max_shard }}) | |
| needs: baseline | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJSON(needs.baseline.outputs.shards) }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| persist-credentials: false | |
| - id: nss-version | |
| run: echo "minimum=$(cat min_version.txt)" >> "$GITHUB_OUTPUT" | |
| - uses: ./.github/actions/nss | |
| with: | |
| minimum-version: ${{ steps.nss-version.outputs.minimum }} | |
| - uses: ./.github/actions/rust | |
| with: | |
| version: nightly | |
| tools: cargo-mutants | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Find mutants | |
| env: | |
| SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} | |
| TIMEOUT: ${{ needs.baseline.outputs.timeout }} | |
| run: | | |
| # shellcheck disable=SC2086 | |
| # Exit codes: 0=success, 1=build/test failure (fail workflow), | |
| # 2=missed, 3=timeout, 4=unviable (suppress, report in summary). | |
| (cargo mutants --shard "$SHARD" --sharding round-robin --baseline=skip --timeout "$TIMEOUT" $MUTANTS_ARGS 2>&1 \ | |
| || { ec=$?; [ $ec -ge 2 ] && [ $ec -le 4 ] && exit 0; exit $ec; }) | tee results.txt | |
| # Some sharded runs get killed by GitHub with error code 143. | |
| # This seems to be a GitHub-internal protection feature that we can't control: | |
| # https://github.com/actions/runner-images/issues/6680 | |
| - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() | |
| with: | |
| name: mutants.out-${{ matrix.shard }} | |
| path: mutants.out | |
| retention-days: 1 | |
| results: | |
| name: Results | |
| if: ${{ !cancelled() }} | |
| needs: mutants | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| pattern: mutants.out-* | |
| path: shards | |
| - name: Merge shard results | |
| run: | | |
| mkdir -p mutants.out | |
| # Move each shard to a subfolder and concatenate result files. | |
| for dir in shards/mutants.out-*; do | |
| shard="${dir##*-}" | |
| mv "$dir" "mutants.out/shard-$shard" | |
| done | |
| for category in caught missed timeout unviable; do | |
| cat mutants.out/shard-*/"$category.txt" 2>/dev/null | sort -u > "mutants.out/$category.txt" || true | |
| rm -f mutants.out/shard-*/"$category.txt" | |
| done | |
| - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| id: upload | |
| with: | |
| name: mutants.out | |
| path: mutants.out | |
| compression-level: 9 | |
| - name: Post step summary | |
| env: | |
| ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} | |
| run: | | |
| { | |
| echo "## Mutation Testing Results" | |
| echo "| Category | Count |" | |
| echo "|----------|------:|" | |
| for category in caught missed timeout unviable; do | |
| count=$(wc -l < "mutants.out/$category.txt" 2>/dev/null | tr -d ' ' || echo 0) | |
| echo "| $category | $count |" | |
| done | |
| echo "" | |
| for category in missed timeout; do | |
| if [ -s "mutants.out/$category.txt" ]; then | |
| echo "### Files with most $category mutants" | |
| echo '```' | |
| # Group by file, count occurrences, show top 10. | |
| cut -d: -f1 "mutants.out/$category.txt" | sort | uniq -c | sort -rn | head -10 | |
| echo '```' | |
| fi | |
| done | |
| echo "" | |
| echo "[Download full results]($ARTIFACT_URL)" | |
| } >> "$GITHUB_STEP_SUMMARY" |