Fix. #4001
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust | |
| on: | |
| push: | |
| branches: [master, thread] | |
| pull_request: | |
| branches: [master, thread] | |
| jobs: | |
| amd64: | |
| runs-on: ubuntu-latest | |
| env: | |
| CARGO_TERM_COLOR: always | |
| # Backstop + coverage: a runaway allocation aborts with a report here | |
| # too (the 16 GB Linux runner survives OOM better, but a named crash | |
| # still beats a thrashing one), and having the limit set exercises the | |
| # cap's init + hot-path compare under the coverage run. | |
| MONORUBY_MALLOC_HARD_LIMIT: 6G | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: "4.0.2" | |
| - uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| toolchain: nightly | |
| - uses: taiki-e/install-action@cargo-llvm-cov | |
| - uses: taiki-e/install-action@nextest | |
| - name: Clone optcarrot | |
| run: git clone https://github.com/mame/optcarrot.git ../optcarrot | |
| - name: Clone ruby/spec and mspec (pinned) | |
| # Pinned to fixed commits so upstream ruby/spec changes can't break CI. | |
| # `bin/test` runs these specs under `set -e`, so a single newly added | |
| # upstream spec that monoruby doesn't pass yet would fail the whole run. | |
| # Bump these SHAs deliberately when adopting newer specs. | |
| env: | |
| SPEC_REV: baf5738ada1709afdbcf222c7609d3379b9d211d | |
| MSPEC_REV: dffcdf7d00612c13465983224203e457a30cf124 | |
| run: | | |
| git init ../spec | |
| git -C ../spec fetch --depth 1 https://github.com/ruby/spec.git "$SPEC_REV" | |
| git -C ../spec checkout FETCH_HEAD | |
| git init ../mspec | |
| git -C ../mspec fetch --depth 1 https://github.com/ruby/mspec.git "$MSPEC_REV" | |
| git -C ../mspec checkout FETCH_HEAD | |
| - name: Clone ruby-bench | |
| run: git clone --depth 1 https://github.com/ruby/ruby-bench.git ../ruby-bench | |
| - name: Install ruby-bench gem dependencies | |
| # bin/ruby-bench-diff requires `erubi`. The other benchmarks | |
| # only use stdlib gems, so a single explicit install is enough. | |
| run: gem install erubi --no-document | |
| - name: Test | |
| run: bin/test | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| # Coverage upload is a non-essential, externally-dependent step: the | |
| # Codecov CLI download/GPG-signature verification fails intermittently | |
| # on Codecov's side (e.g. "Could not verify signature. No public key"), | |
| # which must not fail the whole CI run when the tests themselves pass. | |
| continue-on-error: true | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos | |
| files: lcov.info | |
| fail_ci_if_error: false | |
| darwin: | |
| # checks: write lets the in-step heartbeat publish a `darwin-heartbeat` | |
| # check run that SURVIVES runner death (see the Test step); contents: | |
| # read is for checkout. Everything else needs no token. | |
| permissions: | |
| checks: write | |
| contents: read | |
| # Native Apple Silicon (arm64-apple-darwin) runner. build.rs sets the `jit` | |
| # cfg on arm64 (only `jit_x86` is x86-only), so this builds and runs the | |
| # AArch64 JIT (AsmIR→A64 backend, asmir/compile_stub.rs) — instructions not | |
| # yet ported bail to the VM. It runs the same `bin/test` as the Linux | |
| # `amd64` job (portable head/yjit handling makes it BSD/macOS-safe), so | |
| # the arm64 VM + JIT get the full check scope (unit tests, benchmark diffs, | |
| # optcarrot, ruby-bench, ruby/spec). | |
| # | |
| # SKIP_COV=1: this job does NOT collect coverage. On the GitHub M1 runner, | |
| # llvm-cov coverage instrumentation itself trips a latent VM-layer memory | |
| # bug (a value is freed mid-flight -> String INVALID(255) -> SIGABRT). It is | |
| # independent of the JIT (`--no-jit` crashes the same way) and does not | |
| # reproduce on x86-64, qemu-aarch64, or a local Apple Silicon Mac — so it is | |
| # specific to native-M1 + coverage instrumentation and is tracked | |
| # separately. The Linux `amd64` job owns the Codecov upload. | |
| runs-on: macos-latest | |
| # Backstop: if a hung attempt somehow escapes the per-attempt cap on the | |
| # Test step below, GitHub cancels the job cleanly here (2 attempts x | |
| # 35 min + setup/clone overhead) rather than letting the runner die with | |
| # "lost communication". | |
| timeout-minutes: 95 | |
| env: | |
| CARGO_TERM_COLOR: always | |
| SKIP_COV: "1" | |
| # Drop `gc-stress` (a collection per allocation) on this runner only. | |
| # It was the single biggest multiplier in the run, and the M1 runner | |
| # was finishing right at its per-attempt cap — three commits in a row | |
| # passed only on the second attempt, and the fourth then exhausted | |
| # both. The Linux `amd64` job still runs with it, so every push | |
| # still exercises the GC paths under stress. | |
| SKIP_GC_STRESS: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: "4.0.2" | |
| - name: Install bigdecimal (no longer a default gem since Ruby 3.4) | |
| run: gem install bigdecimal --no-document | |
| - name: Install ruby-bench gem dependencies | |
| # bin/ruby-bench-diff requires `erubi`; matches the `coverage` job. | |
| run: gem install erubi --no-document | |
| - name: Install libffi + pkg-config | |
| # The aarch64-macOS dep block in monoruby/Cargo.toml builds libffi-sys | |
| # with the `system` feature; libffi is keg-only so pkg-config needs its | |
| # prefix on PKG_CONFIG_PATH to find it. | |
| run: | | |
| brew install libffi pkg-config | |
| echo "PKG_CONFIG_PATH=$(brew --prefix libffi)/lib/pkgconfig" >> "$GITHUB_ENV" | |
| - uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| toolchain: nightly | |
| - uses: taiki-e/install-action@nextest | |
| - name: Clone optcarrot | |
| run: git clone https://github.com/mame/optcarrot.git ../optcarrot | |
| - name: Clone ruby/spec and mspec (pinned) | |
| # Pinned to fixed commits so upstream ruby/spec changes can't break CI. | |
| # Keep these SHAs in sync with the `amd64` job above; bump deliberately. | |
| env: | |
| SPEC_REV: baf5738ada1709afdbcf222c7609d3379b9d211d | |
| MSPEC_REV: dffcdf7d00612c13465983224203e457a30cf124 | |
| run: | | |
| git init ../spec | |
| git -C ../spec fetch --depth 1 https://github.com/ruby/spec.git "$SPEC_REV" | |
| git -C ../spec checkout FETCH_HEAD | |
| git init ../mspec | |
| git -C ../mspec fetch --depth 1 https://github.com/ruby/mspec.git "$MSPEC_REV" | |
| git -C ../mspec checkout FETCH_HEAD | |
| - name: Clone ruby-bench | |
| run: git clone --depth 1 https://github.com/ruby/ruby-bench.git ../ruby-bench | |
| - name: Test (VM + AArch64 JIT, arm64-apple-darwin) | |
| # The arm64 JIT/runtime intermittently hangs by exhausting memory: the | |
| # macOS runner OOM-dies ("loses communication") before any stack can be | |
| # captured, and when the runner dies the job's streamed logs AND the | |
| # artifact step are lost as well (confirmed 2026-07-26: a run with the | |
| # in-log heartbeat still left zero forensics) — so past hangs left | |
| # nothing. Guards, in order of importance: | |
| # * MONORUBY_MALLOC_HARD_LIMIT=3G (env below): monoruby's global | |
| # allocator aborts WITH a size report + backtrace the moment an | |
| # allocation would exceed 3 GB. Unlike any polling watchdog this | |
| # also catches a single multi-GB request, and nextest then names | |
| # the aborted test. This is the primary defense. | |
| # * a `darwin-heartbeat` check run updated every ~60 s from inside | |
| # the step (last log lines + memory trajectory + any watchdog | |
| # forensics). It lives on the GitHub side, so it SURVIVES runner | |
| # death — the progress log of record for silent hangs. | |
| # * retry — cap each attempt at 35 min and retry once, so a clean | |
| # hang fails fast (normal pass ~11 min) and the rerun recovers; | |
| # * a background memory watchdog: watches every monoruby AND ruby | |
| # process (bin/test diffs against CRuby and runs mspec) plus the | |
| # system-wide free memory; fires at >3 GB per process / >4 GB | |
| # watched sum / <700 MB free, logs the last [phase] marker + the | |
| # culprit's command line, `sample`s its stack, kills it; | |
| # * a 30-second heartbeat line in the streamed log, and bin/test | |
| # output tee'd to $RUNNER_TEMP + uploaded as an artifact when the | |
| # runner survives (preserves the log of a timed-out attempt that | |
| # the retry then recovered). | |
| # Per-test hangs inside the nextest phase are additionally named and | |
| # killed by .config/nextest.toml (slow-timeout terminate-after). | |
| # The manual .github/workflows/darwin-hang-debug.yml drives the same | |
| # idea in a loop for active reproduction. | |
| uses: nick-fields/retry@v3 | |
| env: | |
| MONORUBY_MALLOC_HARD_LIMIT: 3G | |
| GH_TOKEN: ${{ github.token }} | |
| # Experiment (2026-07-26): all three instrumented runner deaths | |
| # struck inside the nextest phase at unrelated suite positions | |
| # (tests #1438, #1360, #422) with memory healthy 30 s before — | |
| # a fast ambient blow-up, not one culprit test. Default | |
| # parallelism on the M1 runner is 3; halving it tests the | |
| # parallel-interaction hypothesis and lowers peak pressure. | |
| # A normal pass is ~11 min, so the ~+40% nextest time stays | |
| # well under the attempt cap. | |
| NEXTEST_TEST_THREADS: "2" | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 2 | |
| command: | | |
| set -o pipefail | |
| LOG="$RUNNER_TEMP/bin-test-attempt-$(date -u +%H%M%S).log" | |
| FORENSICS="$RUNNER_TEMP/watchdog-forensics.log" | |
| CR_ID_FILE="$RUNNER_TEMP/hb-checkrun-id" | |
| echo "attempt log: $LOG" | |
| # Durable heartbeat: a neutral check run on the head commit, | |
| # PATCHed with the latest progress. It survives runner death. | |
| HB_SHA=$(jq -r '.pull_request.head.sha // .after // empty' "$GITHUB_EVENT_PATH" 2>/dev/null || true) | |
| HB_SHA=${HB_SHA:-$GITHUB_SHA} | |
| hb_post() { | |
| [ -n "${GH_TOKEN:-}" ] || return 0 | |
| if [ ! -s "$CR_ID_FILE" ]; then | |
| curl -sf -X POST -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/$GITHUB_REPOSITORY/check-runs" \ | |
| -d "{\"name\":\"darwin-heartbeat\",\"head_sha\":\"$HB_SHA\",\"status\":\"completed\",\"conclusion\":\"neutral\"}" \ | |
| | jq -r '.id // empty' > "$CR_ID_FILE" 2>/dev/null || true | |
| fi | |
| hbid=$(cat "$CR_ID_FILE" 2>/dev/null || true) | |
| [ -n "$hbid" ] || return 0 | |
| { echo '### in-flight monoruby/ruby processes (nextest passes the test name in argv)' | |
| echo '```' | |
| ps -Ao pid,rss,etime,command | awk 'NR>1 { n=split($4,a,"/"); b=a[n]; if (b ~ /^(monoruby|ruby)/) print }' | head -30 | |
| echo '```' | |
| if [ -s "$FORENSICS" ]; then | |
| echo '### watchdog forensics'; echo '```'; tail -c 18000 "$FORENSICS"; echo '```' | |
| fi | |
| echo '### bin/test log tail'; echo '```'; tail -n 100 "$LOG" 2>/dev/null || true; echo '```' | |
| } > "$RUNNER_TEMP/hb-body.md" | |
| jq -n --arg s "$1" --rawfile t "$RUNNER_TEMP/hb-body.md" \ | |
| '{output:{title:"darwin bin/test heartbeat (survives runner death)",summary:$s,text:($t|.[0:60000])}}' \ | |
| | curl -sf -X PATCH -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/$GITHUB_REPOSITORY/check-runs/$hbid" \ | |
| -d @- > /dev/null 2>&1 || true | |
| } | |
| RSS_LIMIT_KB=3000000 | |
| SUM_LIMIT_KB=4000000 | |
| SYS_FREE_FLOOR_KB=700000 | |
| ( t0=$(date +%s); pass=0 | |
| while true; do | |
| snap=$(ps -Ao pid,rss,comm \ | |
| | awk 'NR>1 { n=split($3,a,"/"); b=a[n]; if (b ~ /^(monoruby|ruby)/) print $1, $2, b }') | |
| sum=0; toppid=0; toprss=0; topcmd=- | |
| if [ -n "$snap" ]; then | |
| sum=$(echo "$snap" | awk '{s+=$2} END{print s+0}') | |
| set -- $(echo "$snap" | awk '{if($2>m){m=$2;p=$1;c=$3}} END{print p+0, m+0, c}') | |
| toppid=$1; toprss=$2; topcmd=${3:--} | |
| fi | |
| # System-wide reclaimable memory (free+inactive+purgeable, 16 KB | |
| # pages on Apple Silicon): the runner dies silently when this | |
| # runs out, whichever process is responsible. | |
| freekb=$(vm_stat | awk -F'[: .]+' \ | |
| '/Pages free/ {f=$3} /Pages inactive/ {i=$3} /Pages purgeable/ {p=$3} END {print (f+i+p)*16}') | |
| if [ "${toprss:-0}" -gt "$RSS_LIMIT_KB" ] || [ "${sum:-0}" -gt "$SUM_LIMIT_KB" ] \ | |
| || { [ "${freekb:-999999999}" -lt "$SYS_FREE_FLOOR_KB" ] && [ "${toppid:-0}" -gt 0 ]; }; then | |
| { echo "==================== MEMORY WATCHDOG fired @ $(date -u +%H:%M:%S) ====================" | |
| echo "sys_free_kb=$freekb sum_rss_kb=$sum top=$topcmd pid=$toppid rss_kb=$toprss" | |
| echo "(caps: per-proc=$RSS_LIMIT_KB sum=$SUM_LIMIT_KB sys-floor=$SYS_FREE_FLOOR_KB)" | |
| echo "---- last phase reached ----"; grep '^\[phase' "$LOG" 2>/dev/null | tail -1 || true | |
| echo "---- watched processes (pid rss_kb etime command) ----" | |
| ps -Ao pid,rss,etime,command | awk 'NR>1 { n=split($4,a,"/"); b=a[n]; if (b ~ /^(monoruby|ruby)/) print }' | |
| echo "---- CULPRIT command line (pid $toppid) ----"; ps -p "$toppid" -o command= 2>/dev/null || true | |
| echo "---- sample pid $toppid (2s) ----"; /usr/bin/sample "$toppid" 2 2>&1 | sed -n '1,160p' || true | |
| echo "---- killing pid $toppid ----"; kill -9 "$toppid" 2>/dev/null || true | |
| } 2>&1 | tee -a "$FORENSICS" | |
| hb_post "WATCHDOG FIRED $(date -u +%H:%M:%S): top=$topcmd rss_kb=${toprss:-0} sys_free_kb=${freekb:-?}" | |
| sleep 8 | |
| fi | |
| pass=$((pass+1)) | |
| if [ $((pass % 8)) -eq 0 ]; then | |
| gtop=$(ps -Ao pid,rss,comm -m 2>/dev/null | awk 'NR==2 {n=split($3,a,"/"); print a[n]"("$2"KB)"}') | |
| diskfree=$(df -k / 2>/dev/null | awk 'NR==2 {print $4}') | |
| hbline=$(printf '[hb %s +%ss] sys_free_kb=%s disk_free_kb=%s watched_sum_kb=%s top=%s(%sKB) global_top=%s tail: %s' \ | |
| "$(date -u +%H:%M:%S)" "$(( $(date +%s) - t0 ))" "${freekb:-?}" "${diskfree:-?}" "$sum" \ | |
| "$topcmd" "${toprss:-0}" "${gtop:--}" \ | |
| "$(tail -c 400 "$LOG" 2>/dev/null | tr '\n' ' ' | tail -c 160)") | |
| echo "$hbline" | |
| # Death leaves at most ~30 s of blind window; the posted | |
| # body names the tests in flight via their argv. | |
| hb_post "$hbline" | |
| fi | |
| sleep 4 | |
| done ) & | |
| WATCHDOG_PID=$! | |
| trap 'kill "$WATCHDOG_PID" 2>/dev/null || true' EXIT | |
| bin/test 2>&1 | tee "$LOG" | |
| - name: Upload darwin test logs (hang forensics) | |
| # Preserves the tee'd bin/test log of every attempt — including a | |
| # first attempt that hit the 25-min cap and was then recovered by the | |
| # retry, whose hang would otherwise leave no trace in a green run. | |
| # (If the runner itself dies, this step never runs; that case is | |
| # covered by the watchdog killing the culprit first.) | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: darwin-bin-test-logs-run${{ github.run_id }}-attempt${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/bin-test-attempt-*.log | |
| if-no-files-found: ignore | |
| retention-days: 14 | |