Simulation Readiness Nightly #171
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Simulation Readiness Nightly | |
| on: | |
| schedule: | |
| - cron: "25 5 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| class-1-nightly: | |
| name: Class-1 Nightly Trends | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: runtimes/ruby | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: "3.4" | |
| bundler-cache: true | |
| - name: Install dependencies | |
| run: bundle install | |
| - name: Run expanded-seed readiness simulation | |
| run: | | |
| set -euo pipefail | |
| SEEDS="11,19,23,31,43,59,71,97,101,131,149,173" | |
| NIGHTLY_ROOT="../../tmp/simulation/nightly" | |
| LEDGER_PATH="${NIGHTLY_ROOT}/run-ledger.jsonl" | |
| TREND_REPORT="${NIGHTLY_ROOT}/nightly-trend-report.json" | |
| ADVISORY_JSON="${NIGHTLY_ROOT}/advisory-status.json" | |
| ADVISORY_MARKDOWN="${NIGHTLY_ROOT}/simulation-advisory-status-$(date -u +%F).md" | |
| TRACE_LOG="${NIGHTLY_ROOT}/trace.jsonl" | |
| mkdir -p "${NIGHTLY_ROOT}" | |
| cat > "${TRACE_LOG}" <<'JSONL' | |
| {"timestamp":"2026-02-22T00:00:00.000Z","role":"simulation_runner","model":"deterministic","method":"nightly","args":[],"kwargs":{},"code":"result = 1","duration_ms":1.0,"generation_attempt":1} | |
| JSONL | |
| run_pack() { | |
| local pack_id="$1" | |
| local pack_path="$2" | |
| local pack_out="${NIGHTLY_ROOT}/${pack_id}" | |
| local fixture_root="${NIGHTLY_ROOT}/fixtures/${pack_id}" | |
| mkdir -p "${pack_out}" "${fixture_root}" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode fixture \ | |
| --seeds "${SEEDS}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/fixture.json" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode replay \ | |
| --seeds "${SEEDS}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/replay-1.json" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode replay \ | |
| --seeds "${SEEDS}" \ | |
| --trace-log "${TRACE_LOG}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/replay-2.json" | |
| jq -e '.gates | to_entries | all(.value.status == "pass")' "${pack_out}/replay-2.json" > /dev/null | |
| } | |
| run_pack_advisory() { | |
| local pack_id="$1" | |
| local pack_path="$2" | |
| local pack_out="${NIGHTLY_ROOT}/${pack_id}" | |
| local fixture_root="${NIGHTLY_ROOT}/fixtures/${pack_id}" | |
| mkdir -p "${pack_out}" "${fixture_root}" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode fixture \ | |
| --seeds "${SEEDS}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/fixture.json" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode replay \ | |
| --seeds "${SEEDS}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/replay-1.json" | |
| bundle exec ruby bin/recurgent-sim \ | |
| --pack "${pack_path}" \ | |
| --mode replay \ | |
| --seeds "${SEEDS}" \ | |
| --trace-log "${TRACE_LOG}" \ | |
| --operational-mode nightly \ | |
| --nightly-trend-report "${TREND_REPORT}" \ | |
| --fixture-root "${fixture_root}" \ | |
| --ledger-path "${LEDGER_PATH}" > "${pack_out}/replay-2.json" | |
| } | |
| run_pack "calculator-core-v1" "../../specs/contract/v1/simulation/scenario-packs/calculator-core-v1.yaml" | |
| run_pack "calculator-edge-v1" "../../specs/contract/v1/simulation/scenario-packs/calculator-edge-v1.yaml" | |
| run_pack_advisory "assistant-continuity-v1" "../../specs/contract/v1/simulation/scenario-packs/assistant-continuity-v1.yaml" | |
| run_pack_advisory "debate-orchestration-v1" "../../specs/contract/v1/simulation/scenario-packs/debate-orchestration-v1.yaml" | |
| bundle exec ruby bin/recurgent-sim-advisory-report \ | |
| --ledger-path "${LEDGER_PATH}" \ | |
| --pack-ids "assistant-continuity-v1,debate-orchestration-v1" \ | |
| --write-json "${ADVISORY_JSON}" \ | |
| --write-markdown "${ADVISORY_MARKDOWN}" > "${NIGHTLY_ROOT}/advisory-analysis.json" | |
| jq -s ' | |
| { | |
| generated_at: (now | todate), | |
| seed_matrix: "'"${SEEDS}"'" | split(",") | map(tonumber), | |
| runs: map({ | |
| run_id, | |
| scenario_pack_id, | |
| mode, | |
| overall_score: .score_vector.overall, | |
| gates: (.gates | with_entries(.value = .value.status)), | |
| baseline_diff_classification: .metrics.baseline_diff_report.classification | |
| }) | |
| } | |
| ' "${LEDGER_PATH}" > "${TREND_REPORT}" | |
| { | |
| echo "### Nightly Simulation Readiness" | |
| echo "" | |
| echo "- Seed matrix: \`${SEEDS}\`" | |
| echo "- Trend report: \`${TREND_REPORT}\`" | |
| echo "- Advisory report: \`${ADVISORY_MARKDOWN}\`" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload nightly readiness artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: simulation-readiness-nightly | |
| path: tmp/simulation/nightly |