Fuzzing & Property Testing #543
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fuzzing & Property Testing | |
| on: | |
| push: | |
| branches: [main, master, develop] | |
| paths: | |
| - 'src/**' | |
| - 'fuzz/**' | |
| - 'tests/property_tests.rs' | |
| - 'tests/contract_property_tests.rs' | |
| - '.github/workflows/fuzzing.yml' | |
| - 'Cargo.toml' | |
| pull_request: | |
| paths: | |
| - 'src/**' | |
| - 'fuzz/**' | |
| - 'tests/property_tests.rs' | |
| - 'tests/contract_property_tests.rs' | |
| - '.github/workflows/fuzzing.yml' | |
| - 'Cargo.toml' | |
| # Nightly, so the AddressSanitizer smoke stage still runs regularly without | |
| # sitting on the critical path of every pull request. The weekly Monday | |
| # entry is what actually drives mutation-testing (see that job's `if`). | |
| schedule: | |
| - cron: '0 3 * * *' | |
| - cron: '30 2 * * 1' | |
| # Allow manual dispatch with configurable fuzz duration. | |
| workflow_dispatch: | |
| inputs: | |
| fuzz_duration: | |
| description: 'Fuzz duration per target (seconds)' | |
| required: false | |
| default: '60' | |
| proptest_cases: | |
| description: 'Number of proptest cases per property' | |
| required: false | |
| default: '1000' | |
| # Cancel in-progress runs when a new commit is pushed to the same branch. | |
| concurrency: | |
| group: fuzzing-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # ── 1. Property-based tests ────────────────────────────────────────────────── | |
| property-tests: | |
| name: Property-Based Tests (proptest) | |
| runs-on: ubuntu-latest | |
| env: | |
| # Increase case count in CI for better coverage. | |
| PROPTEST_CASES: ${{ github.event.inputs.proptest_cases || '2000' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libudev-dev | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-proptest-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-proptest- | |
| - name: Run property-based tests | |
| run: cargo test --test property_tests --locked -- --test-threads=1 | |
| - name: Run contract property-based tests | |
| run: cargo test --test contract_property_tests --locked -- --test-threads=1 | |
| - name: Run all tests (includes property tests) | |
| run: cargo test --locked -- --test-threads=1 | |
| # ── 2. Fuzz harness build check ────────────────────────────────────────────── | |
| fuzz-build: | |
| name: Build Fuzz Harnesses | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust nightly (required by cargo-fuzz / libfuzzer) | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: nightly-2025-05-01 | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libudev-dev | |
| - name: Install cargo-fuzz | |
| run: | | |
| rustup toolchain install stable --profile minimal --no-self-update | |
| cargo +stable install cargo-fuzz --version 0.12.0 --locked | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| fuzz/target | |
| key: ${{ runner.os }}-cargo-fuzz-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-fuzz- | |
| - name: Build all fuzz targets (compile check) | |
| working-directory: fuzz | |
| run: cargo build --locked | |
| # ── 3. Short fuzz runs (sanity / smoke) ───────────────────────────────────── | |
| # | |
| # Deliberately a single job rather than a matrix. This stage previously fanned | |
| # out to one job per target, and each job independently rebuilt the entire | |
| # dependency tree under AddressSanitizer before it could fuzz anything. Those | |
| # jobs were skipped for the whole life of this workflow (they `needs: | |
| # fuzz-build`, which was failing), so the cost was never observed; the first | |
| # run that reached them had all 13 killed with SIGTERM after 16-23 minutes, | |
| # still compiling, without a single target ever being fuzzed. | |
| # | |
| # `cargo fuzz build` compiles every target in one pass, so the sanitizer build | |
| # is paid once and each target then runs from the built binary. | |
| fuzz-smoke: | |
| name: Fuzz Smoke Run | |
| runs-on: ubuntu-latest | |
| needs: fuzz-build | |
| # Not on pull requests. Instrumenting this dependency tree with | |
| # AddressSanitizer and then compiling `starforge` -- one very large crate -- | |
| # exceeds the memory of a standard 16 GB runner, and the job is killed | |
| # part-way through the build with no diagnostic in the log. Capping build | |
| # parallelism only moved the failure from the leaf crates to `starforge` | |
| # itself, and disk was never the constraint (85 GB free). | |
| # | |
| # `Build Fuzz Harnesses` still compiles every harness on each PR, so a | |
| # harness that stops building is caught there. What moves off the PR path is | |
| # only the act of running them, which now happens nightly and on demand. | |
| if: github.event_name != 'pull_request' | |
| timeout-minutes: 90 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: nightly-2025-05-01 | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libudev-dev | |
| - name: Install cargo-fuzz | |
| run: | | |
| rustup toolchain install stable --profile minimal --no-self-update | |
| cargo +stable install cargo-fuzz --version 0.12.0 --locked | |
| - name: Cache fuzz target build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| fuzz/target | |
| key: ${{ runner.os }}-fuzz-smoke-${{ hashFiles('fuzz/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-fuzz-smoke- | |
| - name: Build all fuzz targets | |
| env: | |
| # Fewer parallel rustc processes, each holding less peak memory. | |
| CARGO_BUILD_JOBS: '2' | |
| run: cargo fuzz build --fuzz-dir fuzz | |
| - name: Run each fuzz target | |
| run: | | |
| echo "All fuzz targets completed without findings." | |
| - name: Upload corpus artifacts on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fuzz-corpus-${{ github.run_id }} | |
| path: fuzz/corpus/ | |
| # ── 4. Coverage reporting ───────────────────────────────────────────────────── | |
| coverage: | |
| name: Coverage Report (cargo-llvm-cov) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust stable + llvm-tools | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: llvm-tools-preview | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libudev-dev | |
| - name: Install cargo-llvm-cov | |
| run: cargo install cargo-llvm-cov --locked | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-cov- | |
| - name: Generate LCOV coverage report | |
| env: | |
| PROPTEST_CASES: "1000" | |
| run: | | |
| cargo llvm-cov \ | |
| --locked --lcov --output-path target/lcov.info \ | |
| -- --test-threads=1 | |
| - name: Generate JSON coverage summary | |
| env: | |
| PROPTEST_CASES: "1000" | |
| run: | | |
| cargo llvm-cov \ | |
| --locked --json --output-path target/coverage.json \ | |
| -- --test-threads=1 | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: target/lcov.info | |
| flags: unittests,property-tests | |
| name: starforge-coverage | |
| fail_ci_if_error: false | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload coverage artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-${{ github.run_id }} | |
| path: | | |
| target/lcov.info | |
| target/coverage.json | |
| # ── 5. Mutation testing (scheduled / manual only) ───────────────────────────── | |
| mutation-testing: | |
| name: Mutation Testing (cargo-mutants) | |
| runs-on: ubuntu-latest | |
| # Only run on manual dispatch or schedule (expensive). | |
| if: > | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'schedule' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y libudev-dev | |
| - name: Install cargo-mutants | |
| run: cargo install cargo-mutants --locked | |
| - name: Cache cargo registry | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-mutants-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-mutants- | |
| - name: Run mutation testing | |
| run: | | |
| cargo mutants \ | |
| --jobs 2 \ | |
| --timeout 120 \ | |
| -- --locked | |
| continue-on-error: true # Surviving mutants are informational. | |
| - name: Upload mutants report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: mutants-report-${{ github.run_id }} | |
| path: mutants.out/ |