Nightly Differential Fuzz #76
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Differential Fuzz | |
| on: | |
| schedule: | |
| - cron: "0 14 * * *" # 22:00 UTC+8 | |
| workflow_dispatch: | |
| inputs: | |
| rounds: | |
| description: "Number of fuzz rounds" | |
| default: "10000" | |
| stability-runs: | |
| description: "Stability check reruns per case" | |
| default: "3" | |
| concurrency: | |
| group: nightly-diff-fuzz-${{ github.ref }} | |
| cancel-in-progress: false | |
| # The default workflow token is read-only on this repo, so the two | |
| # "Open issue on ..." steps below were failing with | |
| # "Resource not accessible by integration (createIssue)" — swallowed by | |
| # continue-on-error, so a divergence run went red with no issue filed. | |
| # issues:write lets gh issue create succeed; contents:read keeps checkout working. | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| differential-fuzz: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 355 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: pine-go/go.mod | |
| cache-dependency-path: pine-go/go.sum | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.13" | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: "25" | |
| cache: maven | |
| - name: Install C++ build deps | |
| # ci-apt-install.sh retries with a per-attempt timeout so one slow | |
| # mirror rotation doesn't burn the whole budget (#125, #164). cmake / | |
| # g++-13 / make are preinstalled on the runner image (apt's cmake is | |
| # older and shadowed by PATH anyway; #164's fatal 11.2 MB download | |
| # was exactly that dead weight) — assert them instead of installing. | |
| run: | | |
| bash scripts/ci-apt-install.sh libluajit-5.1-dev libcurl4-openssl-dev | |
| cmake --version | head -1 | |
| g++-13 --version | head -1 | |
| - name: Build Go binary | |
| run: go build -o pineapple-run ./cmd/pineapple-run/ | |
| working-directory: pine-go | |
| - name: Build Java engine | |
| run: mvn package -B -q -DskipTests | |
| working-directory: pine-java | |
| - name: Codegen schema parity gate (also builds C++ binaries) | |
| # Runs cross-validate Section 1 (schema + apple_generated/ byte-level | |
| # diff across Go/Java/C++). PR-time CI also runs this, but nightly | |
| # picks it up as a low-cost guard against three-way drift in | |
| # operators.py / markers.py / resources.py / __init__.py landing | |
| # silently between PR runs (audit gap H5). | |
| # | |
| # Side effect we deliberately rely on: _prebuild.sh builds the full | |
| # set of pine-cpp binaries (pineapple-run, codegen, server, dag, | |
| # cause-chain-probe) into pine-cpp/build/. That covers the C++ | |
| # binary used by the differential-fuzz step below, so no separate | |
| # "Build C++ engine" step is needed. | |
| run: bash scripts/cross-validate.sh 1 | |
| - name: Run differential fuzz | |
| timeout-minutes: 350 | |
| run: | | |
| set -o pipefail | |
| ROUNDS="${{ inputs.rounds || '10000' }}" | |
| EXTRA_FLAGS="" | |
| rc=0 | |
| timeout 340m python3 scripts/differential-fuzz.py \ | |
| --rounds "$ROUNDS" \ | |
| --stability-runs ${{ inputs.stability-runs || '3' }} \ | |
| --go-bin pine-go/pineapple-run \ | |
| --cpp-bin pine-cpp/build/pineapple-run \ | |
| --engines go,java,cpp \ | |
| --save-dir /tmp/diff-fuzz-nightly \ | |
| $EXTRA_FLAGS \ | |
| | tee diff-fuzz-output.log || rc=$? | |
| echo "$rc" > /tmp/fuzz-exit-code | |
| exit "$rc" | |
| - name: Evaluate results | |
| if: always() | |
| id: evaluate | |
| run: | | |
| FAIL_COUNT=$(grep -oP 'FAIL: +\K[0-9]+' diff-fuzz-output.log 2>/dev/null | tail -1 || echo 0) | |
| UNSTABLE_COUNT=$(grep -oP 'UNSTABLE: +\K[0-9]+' diff-fuzz-output.log 2>/dev/null | tail -1 || echo 0) | |
| FUZZ_EXIT=$(cat /tmp/fuzz-exit-code 2>/dev/null || echo 1) | |
| # `grep -c` already prints 0 on no match; use `|| true` to keep set-e | |
| # happy without appending a second "0" to the captured value. | |
| HAS_RESULTS=$(grep -c '^Results:' diff-fuzz-output.log 2>/dev/null || true) | |
| FAIL_COUNT=${FAIL_COUNT:-0} | |
| UNSTABLE_COUNT=${UNSTABLE_COUNT:-0} | |
| HAS_RESULTS=${HAS_RESULTS:-0} | |
| { | |
| echo "### Final Status" | |
| echo "" | |
| echo '```text' | |
| tail -30 diff-fuzz-output.log 2>/dev/null || echo "(no fuzz output captured)" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if [[ "$HAS_RESULTS" -gt 0 ]]; then | |
| # Fuzz produced a results summary. Trust parsed counts. | |
| if [[ "$FAIL_COUNT" -gt 0 || "$UNSTABLE_COUNT" -gt 0 ]]; then | |
| echo "has_divergences=true" >> "$GITHUB_OUTPUT" | |
| echo "fail_count=$FAIL_COUNT" >> "$GITHUB_OUTPUT" | |
| echo "unstable_count=$UNSTABLE_COUNT" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_divergences=false" >> "$GITHUB_OUTPUT" | |
| if [[ "$FUZZ_EXIT" -ne 0 ]]; then | |
| echo "⚠️ Fuzz finished with 0 fail/unstable but exit=$FUZZ_EXIT — investigate" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| fi | |
| else | |
| # No results summary → fuzz crashed before completion. Infra failure path. | |
| echo "has_divergences=false" >> "$GITHUB_OUTPUT" | |
| echo "⚠️ Fuzz did not produce a results summary (exit=$FUZZ_EXIT) — infrastructure failure" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Package divergences | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| run: | | |
| cd /tmp/diff-fuzz-nightly | |
| tar czf /tmp/diff-fuzz-divergences.tar.gz \ | |
| divergence_*/ unstable_*/ 2>/dev/null || true | |
| - name: Upload divergences | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diff-fuzz-divergences-${{ github.run_number }} | |
| path: /tmp/diff-fuzz-divergences.tar.gz | |
| retention-days: 14 | |
| - name: Build C++ TSan binary | |
| # Only when a divergence was found. The main fuzz runs a Release binary | |
| # (a TSan build is 5-10x slower and would blow the job timeout across | |
| # 10000 rounds). But a C++ divergence — especially a crash (cpp_rc!=0) | |
| # under data_parallel / multi-root parallel — is very likely a data | |
| # race that Release reports as nothing but a non-zero exit. So we build | |
| # TSan once, here, to replay just the saved cases with full diagnostics. | |
| # Same recipe as scripts/cpp-tsan-smoke.sh. | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| run: | | |
| cmake -S pine-cpp -B pine-cpp/build-tsan \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DCMAKE_CXX_FLAGS="-fsanitize=thread -fno-omit-frame-pointer -O1 -g" \ | |
| -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread" \ | |
| -DPINE_USE_JEMALLOC=OFF | |
| cmake --build pine-cpp/build-tsan --target pineapple-run -j2 | |
| - name: TSan replay divergences | |
| # Replay every saved divergence/unstable case under the TSan binary, | |
| # many times each, to surface the data race head (WARNING + read/write | |
| # stacks) that the Release fuzz only saw as a bare non-zero exit. The | |
| # full report is appended to the step summary and saved as an artifact | |
| # so a fix has the stacks it needs instead of waiting for the next | |
| # narrow-window CI recurrence. | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| continue-on-error: true | |
| env: | |
| # halt_on_error=0: keep going so one case's race doesn't stop the | |
| # sweep; second_deadlock_stack surfaces both sides of a lock-order race. | |
| TSAN_OPTIONS: "halt_on_error=0 second_deadlock_stack=1" | |
| run: | | |
| TSAN_BIN=pine-cpp/build-tsan/pineapple-run | |
| REPORT=/tmp/tsan-replay-report.log | |
| : > "$REPORT" | |
| found_race=0 | |
| for case_dir in /tmp/diff-fuzz-nightly/divergence_* /tmp/diff-fuzz-nightly/unstable_*; do | |
| [ -d "$case_dir" ] || continue | |
| name=$(basename "$case_dir") | |
| # Progress to the CI log only — do NOT write the header into | |
| # $REPORT. $REPORT must stay empty until an actual race is caught | |
| # so "non-empty == race found" holds for the issue-body check | |
| # below (most divergences are data-dependent output mismatches, | |
| # not races, and would otherwise false-report a race). | |
| echo "=== TSan replay: $name (40 runs) ===" | |
| for run in $(seq 1 40); do | |
| err=$("$TSAN_BIN" -config "$case_dir/config.json" -request "$case_dir/request.json" 2>&1 >/dev/null) || true | |
| if echo "$err" | grep -q 'ThreadSanitizer'; then | |
| found_race=1 | |
| { | |
| echo "--- $name run $run: ThreadSanitizer report ---" | |
| echo "$err" | |
| echo "" | |
| } >> "$REPORT" | |
| # one full report per case is enough to fix; stop hammering it | |
| break | |
| fi | |
| done | |
| done | |
| { | |
| echo "### TSan replay of divergence cases" | |
| echo "" | |
| if [ "$found_race" -eq 1 ]; then | |
| echo "ThreadSanitizer surfaced a race during replay. Full stacks:" | |
| echo '```' | |
| cat "$REPORT" | |
| echo '```' | |
| else | |
| echo "No ThreadSanitizer report across replays (the divergence may be non-race: data-dependent output mismatch, or a window too narrow even for 40x TSan replays). Release-level case outputs are in the divergence artifact." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload TSan replay report | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: diff-fuzz-tsan-replay-${{ github.run_number }} | |
| path: /tmp/tsan-replay-report.log | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| - name: Open issue on divergence | |
| if: ${{ !cancelled() && steps.evaluate.outputs.has_divergences == 'true' }} | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| FAIL_COUNT="${{ steps.evaluate.outputs.fail_count }}" | |
| UNSTABLE_COUNT="${{ steps.evaluate.outputs.unstable_count }}" | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| ARTIFACT_URL="${RUN_URL}#artifacts" | |
| gh issue create \ | |
| --title "Nightly diff-fuzz: ${FAIL_COUNT} failures, ${UNSTABLE_COUNT} unstable ($(date -u +%Y-%m-%d))" \ | |
| --label "bug,fuzz" \ | |
| --body "$(cat <<EOF | |
| ## Nightly Differential Fuzz Divergence | |
| | | | | |
| |---|---| | |
| | **Run** | ${RUN_URL} | | |
| | **Date** | $(date -u +%Y-%m-%d) | | |
| | **Failures** | ${FAIL_COUNT} | | |
| | **Unstable** | ${UNSTABLE_COUNT} | | |
| | **Artifact** | [diff-fuzz-divergences-${{ github.run_number }}](${ARTIFACT_URL}) | | |
| ### Reproduce locally | |
| 1. Download and extract the artifact: | |
| \`\`\`bash | |
| # Download from: ${ARTIFACT_URL} | |
| tar xzf diff-fuzz-divergences.tar.gz | |
| cd divergence_NNNNNN/ # pick a case | |
| \`\`\` | |
| 2. Run each engine on the case: | |
| \`\`\`bash | |
| # Go | |
| cd pine-go && go run ./cmd/pineapple-run/ -config /path/to/config.json -request /path/to/request.json | |
| # Java | |
| cd pine-java && mvn -q exec:java -Dexec.mainClass="page.liam.pine.Main" -Dexec.args="-config /path/to/config.json -request /path/to/request.json" | |
| # C++ | |
| cd pine-cpp && ./build/pineapple-run -config /path/to/config.json -request /path/to/request.json | |
| \`\`\` | |
| 3. Compare outputs. Each case directory contains: | |
| - \`config.json\` — pipeline configuration | |
| - \`request.json\` — input request | |
| - \`info.txt\` — which engines diverged and their return codes | |
| - \`{go,java,cpp}_output.json\` — actual output from each engine | |
| ### Last 30 lines of fuzz output | |
| \`\`\` | |
| $(tail -30 diff-fuzz-output.log) | |
| \`\`\` | |
| ### TSan replay | |
| $(if [ -s /tmp/tsan-replay-report.log ]; then | |
| echo "ThreadSanitizer surfaced a race when replaying the saved case(s) under the TSan binary:" | |
| echo '```' | |
| head -120 /tmp/tsan-replay-report.log | |
| echo '```' | |
| echo "Full report: artifact \`diff-fuzz-tsan-replay-${{ github.run_number }}\`." | |
| else | |
| echo "TSan replay produced no race report (divergence may be a data-dependent output mismatch rather than a race, or the window is too narrow even under 40x TSan replay)." | |
| fi) | |
| EOF | |
| )" | |
| - name: Open issue on workflow failure | |
| if: ${{ (failure() || cancelled()) && steps.evaluate.outputs.has_divergences != 'true' }} | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| gh issue create \ | |
| --title "Nightly diff-fuzz: workflow failed ($(date -u +%Y-%m-%d))" \ | |
| --label "bug,infra" \ | |
| --body "$(cat <<EOF | |
| ## Nightly Differential Fuzz Workflow Failure | |
| The nightly diff-fuzz workflow failed due to an infrastructure error | |
| (build failure, dependency issue, or script crash). | |
| | | | | |
| |---|---| | |
| | **Run** | ${RUN_URL} | | |
| | **Date** | $(date -u +%Y-%m-%d) | | |
| Check the [workflow logs](${RUN_URL}) for details. | |
| EOF | |
| )" |