test: complete Wasmtime parity validation

Author: NathanFlurry

.github/workflows/ci-nightly.yml

@@ -9,7 +9,7 @@ jobs:
   xfstests:
     name: "xfstests (${{ matrix.wasm_backend }}, ${{ matrix.storage_backend }})"
     runs-on: [self-hosted, agentos-builder]
-    timeout-minutes: 360
+    timeout-minutes: 420
     strategy:
       fail-fast: false
       matrix:
@@ -20,44 +20,124 @@ jobs:
       - uses: pnpm/action-setup@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 22
+          node-version: 24
       - uses: dtolnay/rust-toolchain@stable
         with:
           targets: wasm32-wasip1
+      - name: Materialize the pinned docs theme
+        run: |
+          rm -rf /tmp/docs-theme
+          git clone https://github.com/rivet-dev/docs-theme.git /tmp/docs-theme
+          git -C /tmp/docs-theme checkout 450c498555135098c6a927adfdf13458be9be22a
+          rm -rf website/vendor/theme && mkdir -p website/vendor
+          cp -r /tmp/docs-theme/packages/theme website/vendor/theme
       - run: pnpm install --frozen-lockfile
       - name: Run the pinned filesystem conformance corpus
         run: make -C tests/xfstests run
         env:
           XFSTESTS_WASM_BACKENDS: ${{ matrix.wasm_backend }}
           XFSTESTS_BACKENDS: ${{ matrix.storage_backend }}
           XFSTESTS_CONCURRENCY: '2'
-      - name: Run the 1,000-file multi-process directory stress gate
-        if: matrix.storage_backend == 'chunked_local'
-        run: |
-          cargo test --release -p agentos-native-sidecar \
-            --test xfstests_correctness \
-            xfstests_wasi_dirstress_process_matrix -- \
-            --ignored --exact --nocapture --test-threads=1
-        env:
-          AGENTOS_TEST_WASM_BACKEND: ${{ matrix.wasm_backend }}
-          XFSTESTS_ROOT: ${{ github.workspace }}/tests/xfstests/.work/xfstests
-          AGENTOS_V8_BRIDGE_PREBUILT_DIR: ${{ github.workspace }}/tests/xfstests/.cache/v8-bridge
-          CARGO_TARGET_DIR: ${{ github.workspace }}/tests/xfstests/.cache/cargo-target
-          CARGO_BUILD_JOBS: '1'
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: xfstests-${{ matrix.wasm_backend }}-${{ matrix.storage_backend }}
           path: tests/xfstests/report
           if-no-files-found: warn
 
+  xfstests-endurance:
+    name: "xfstests endurance (${{ matrix.wasm_backend }})"
+    runs-on: [self-hosted, agentos-builder]
+    timeout-minutes: 480
+    strategy:
+      fail-fast: false
+      matrix:
+        wasm_backend: [v8, wasmtime]
+    env:
+      AGENTOS_TEST_WASM_BACKEND: ${{ matrix.wasm_backend }}
+      XFSTESTS_ROOT: ${{ github.workspace }}/tests/xfstests/.work/xfstests
+      XFSTESTS_TEST_TIMEOUT_SECONDS: '3600'
+      AGENTOS_V8_BRIDGE_PREBUILT_DIR: ${{ github.workspace }}/tests/xfstests/.cache/v8-bridge
+      CARGO_TARGET_DIR: ${{ github.workspace }}/tests/xfstests/.cache/cargo-target
+      CARGO_BUILD_JOBS: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-wasip1
+      - name: Materialize the pinned docs theme
+        run: |
+          rm -rf /tmp/docs-theme
+          git clone https://github.com/rivet-dev/docs-theme.git /tmp/docs-theme
+          git -C /tmp/docs-theme checkout 450c498555135098c6a927adfdf13458be9be22a
+          rm -rf website/vendor/theme && mkdir -p website/vendor
+          cp -r /tmp/docs-theme/packages/theme website/vendor/theme
+      - run: pnpm install --frozen-lockfile
+      - name: Stage the pinned corpus and helper binaries
+        run: env -u CARGO_TARGET_DIR make -C tests/xfstests helpers
+      - name: Build the V8 bridge used by the shared harness
+        run: node packages/build-tools/scripts/build-v8-bridge.mjs --out-dir tests/xfstests/.cache/v8-bridge
+      - name: Run the 1,000-file multi-process directory stress gate
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_dirstress_process_matrix -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full parallel ENOSPC race
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_parallel_enospc_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full insert-range reproduction workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_insert_range_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full 162,000-iteration looptest workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_looptest_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full 10,000-file rewinddir workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_rewinddir_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full 10,000-file open-unlink workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_open_unlink_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full 4,000-file seekdir/getdents workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_seekdir_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+      - name: Run the full 5,000-file readdir/rename workload
+        run: |
+          cargo test --release -p agentos-native-sidecar \
+            --test xfstests_correctness \
+            xfstests_wasi_readdir_rename_endurance_probe -- \
+            --ignored --exact --nocapture --test-threads=1
+
   nightly:
     name: Nightly runtime validation
     runs-on: [self-hosted, agentos-builder]
     env:
       # Cargo validation and the shared sidecar build are explicit below.
       # Do not let TypeScript dependency builds compile a second Rust copy.
       AGENTOS_SKIP_NATIVE_META_BUILD: '1'
+      XFSTESTS_ROOT: ${{ github.workspace }}/tests/xfstests/.work/xfstests
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
@@ -84,18 +164,38 @@ jobs:
             sudo apt-get update
             sudo apt-get install --yes cmake
           fi
+      - name: Materialize the pinned docs theme
+        run: |
+          rm -rf /tmp/docs-theme
+          git clone https://github.com/rivet-dev/docs-theme.git /tmp/docs-theme
+          git -C /tmp/docs-theme checkout 450c498555135098c6a927adfdf13458be9be22a
+          rm -rf website/vendor/theme && mkdir -p website/vendor
+          cp -r /tmp/docs-theme/packages/theme website/vendor/theme
       - run: pnpm install --frozen-lockfile
       - run: make -C toolchain commands
-      - run: make -C toolchain cmd/duckdb
+      - run: make -C toolchain cmd/duckdb cmd/vim
       - run: make -C toolchain codex
       - run: make -C toolchain/c pthread-conformance-wasm pthread-benchmark-wasm
       - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require
+      - name: Stage pinned xfstests and its WASM helper corpus
+        run: make -C tests/xfstests helpers XFSTESTS_BUILD_NATIVE_COMMANDS=0
       - name: Use stable Rust for repository validation
         run: echo "RUSTUP_TOOLCHAIN=stable" >> "$GITHUB_ENV"
       # Browser support is retained in-tree but disabled during the unified
       # native sidecar reactor migration, so it must not gate native CI.
       - run: cargo check --workspace --exclude agentos-sidecar-browser --exclude agentos-native-sidecar-browser
       - run: cargo clippy --workspace --exclude agentos-sidecar-browser --exclude agentos-native-sidecar-browser --all-targets -- -D warnings
+      - name: Build JavaScript workspace dependencies
+        run: |
+          pnpm exec turbo build \
+            --only \
+            --concurrency=4 \
+            --filter='!@rivet-dev/agentos-website' \
+            --filter='!@agentos-software/codex' \
+            --filter='!@rivet-dev/agentos-browser' \
+            --filter='!@rivet-dev/agentos-runtime-browser' \
+            --filter='!@rivet-dev/agentos-example-browser-terminal' \
+            --filter='!@rivet-dev/agentos-playground'
       - run: pnpm check-types
       - name: Build sidecars once for all TypeScript runtime suites
         run: cargo build -p agentos-sidecar -p agentos-native-sidecar

.github/workflows/ci.yml

@@ -50,6 +50,7 @@ jobs:
               --filter='!@agentos-software/codex' \
               --filter='!@rivet-dev/agentos-browser' \
               --filter='!@rivet-dev/agentos-runtime-browser' \
+              --filter='!@rivet-dev/agentos-example-browser-terminal' \
               --filter='!@rivet-dev/agentos-playground'
           else
             npx turbo build \
@@ -58,6 +59,7 @@ jobs:
               --filter='!@agentos-software/codex' \
               --filter='!@rivet-dev/agentos-browser' \
               --filter='!@rivet-dev/agentos-runtime-browser' \
+              --filter='!@rivet-dev/agentos-example-browser-terminal' \
               --filter='!@rivet-dev/agentos-playground'
           fi
       # The Codex adapter is ordinary TypeScript, while its executable is a
@@ -74,13 +76,15 @@ jobs:
       - run: node scripts/verify-fixed-versions.mjs
       - run: node --test scripts/check-layout.test.mjs
       - run: pnpm check-layout
+      - run: node --test website/scripts/gen-registry.test.mjs
       - run: node --test scripts/check-rustfmt.test.mjs
       - run: node scripts/check-rustfmt.mjs
       - run: pnpm check-types
       - name: Run fast package unit tests
         run: |
           pnpm --parallel --aggregate-output \
             --filter '@rivet-dev/agentos-build-tools' \
+            --filter '@rivet-dev/agentos-runtime-benchmarks' \
             --filter '@rivet-dev/agentos-toolchain' \
             --filter '@rivet-dev/agentos-runtime-sidecar' \
             test
@@ -121,24 +125,47 @@ jobs:
           workspaces: toolchain -> target
           key: wasm-commands-${{ hashFiles('toolchain/Cargo.lock') }}
       - run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...'
-      - run: make -C toolchain commands
+      - name: Build the bounded PR command corpus
+        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        run: make -C toolchain pr-commands
+      - name: Build the complete command corpus
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        run: make -C toolchain commands
       # DuckDB and Vim are intentionally outside the default command set because
       # they are heavy builds, but the runtime/parity suites require their real
       # command artifacts.
       - name: Build required heavy integration commands
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
         run: make -C toolchain cmd/duckdb cmd/vim
+      - name: Build the pinned Codex WASI artifacts required by the software suite
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        run: make -C toolchain codex-required
       - name: Build mandatory threaded-WASM conformance fixtures
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
         run: make -C toolchain/c pthread-conformance-wasm pthread-benchmark-wasm
       - name: Stage native/WASM C parity fixtures
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
         run: make -C toolchain/c conformance-artifacts
-      - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require
+      - name: Stage the bounded PR command corpus
+        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require-coreutils
+      - name: Stage the complete command corpus
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require
       - uses: actions/upload-artifact@v4
         with:
           name: wasm-commands
           path: packages/runtime-core/commands
           retention-days: 1
           if-no-files-found: error
       - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
+        with:
+          name: codex-wasi
+          path: software/codex/wasm
+          if-no-files-found: error
+      - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
         with:
           name: wasm-thread-fixtures
           path: |
@@ -147,6 +174,7 @@ jobs:
             toolchain/c/build/exec_variants
           if-no-files-found: error
       - uses: actions/upload-artifact@v4
+        if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')
         with:
           name: wasm-c-parity-fixtures
           path: toolchain/c/build/conformance
@@ -329,7 +357,7 @@ jobs:
   required:
     name: CI / Required
     if: ${{ always() && (github.event_name != 'pull_request' || github.base_ref == 'main' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci')) }}
-    needs: [checks, wasm-commands, rust, core-pr, runtime-core-pr, actor-pr]
+    needs: [checks, wasm-commands, rust, core-pr, runtime-core-pr, actor-pr, wasm-backend-matrix]
     runs-on: ubuntu-latest
     steps:
       - name: Require every PR gate
@@ -340,6 +368,8 @@ jobs:
           CORE_RESULT: ${{ needs.core-pr.result }}
           RUNTIME_CORE_RESULT: ${{ needs.runtime-core-pr.result }}
           ACTOR_RESULT: ${{ needs.actor-pr.result }}
+          WASM_BACKEND_RESULT: ${{ needs.wasm-backend-matrix.result }}
+          EXPECT_WASM_BACKEND_MATRIX: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'aggregate-ci') }}
         run: |
           for result in \
             "$CHECKS_RESULT" \
@@ -353,6 +383,10 @@ jobs:
               exit 1
             fi
           done
+          if [ "$EXPECT_WASM_BACKEND_MATRIX" = true ] && [ "$WASM_BACKEND_RESULT" != success ]; then
+            echo "required dual-backend CI job did not succeed: $WASM_BACKEND_RESULT" >&2
+            exit 1
+          fi
 
   wasm-backend-matrix:
     name: "WASM backend (${{ matrix.backend }})"
@@ -368,25 +402,39 @@ jobs:
       AGENTOS_TEST_WASM_BACKEND: ${{ matrix.backend }}
       AGENTOS_E2E_NETWORK: '1'
       AGENT_OS_CLIENT_ALLOW_E2E_SKIPS: '0'
-      AGENTOS_SIDECAR_BIN: ${{ github.workspace }}/target/release/agentos-native-sidecar
+      AGENTOS_SIDECAR_BIN: ${{ github.workspace }}/target/release/agentos-sidecar
       AGENTOS_WASMTIME_WORKER_PATH: ${{ github.workspace }}/target/release/agentos-native-sidecar
       AGENTOS_WASM_COMMANDS_DIR: ${{ github.workspace }}/packages/runtime-core/commands
       AGENTOS_C_WASM_COMMANDS_DIR: ${{ github.workspace }}/toolchain/c/build/conformance
+      XFSTESTS_ROOT: ${{ github.workspace }}/tests/xfstests/.work/xfstests
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 22
+          node-version: 24
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
           key: wasm-backend-${{ matrix.backend }}
+      - name: Materialize the pinned docs theme
+        run: |
+          rm -rf /tmp/docs-theme
+          git clone https://github.com/rivet-dev/docs-theme.git /tmp/docs-theme
+          git -C /tmp/docs-theme checkout 450c498555135098c6a927adfdf13458be9be22a
+          rm -rf website/vendor/theme && mkdir -p website/vendor
+          cp -r /tmp/docs-theme/packages/theme website/vendor/theme
       - run: pnpm install --frozen-lockfile
       - uses: actions/download-artifact@v4
         with:
           name: wasm-commands
           path: packages/runtime-core/commands
+      - uses: actions/download-artifact@v4
+        with:
+          name: codex-wasi
+          path: software/codex/wasm
+      - name: Restore Codex WASI executable modes
+        run: chmod 0755 software/codex/wasm/codex software/codex/wasm/codex-exec
       - uses: actions/download-artifact@v4
         with:
           name: wasm-thread-fixtures
@@ -395,22 +443,32 @@ jobs:
         with:
           name: wasm-c-parity-fixtures
           path: toolchain/c/build/conformance
+      - name: Restore native C parity executable modes
+        run: find toolchain/c/build/conformance/native -maxdepth 1 -type f -exec chmod 0755 '{}' +
       - name: Restore and package the validated command corpus
         run: |
           mkdir -p toolchain/target/wasm32-wasip1/release/commands
           cp -a packages/runtime-core/commands/. toolchain/target/wasm32-wasip1/release/commands/
           node packages/runtime-core/scripts/copy-wasm-commands.mjs --require
+          pnpm --filter @agentos-software/manifest build
+          pnpm --filter @rivet-dev/agentos-toolchain build
           pnpm --filter @agentos-software/coreutils build:runtime
-          pnpm --filter '@agentos-software/common...' build
+          pnpm --filter '@agentos-software/*' build
+      - name: Stage pinned xfstests and its WASM helper corpus
+        run: make -C tests/xfstests helpers XFSTESTS_BUILD_NATIVE_COMMANDS=0
       - name: Build the release sidecar and public test clients
         run: |
-          cargo build --release -p agentos-native-sidecar
+          cargo build --release -p agentos-sidecar -p agentos-native-sidecar
           pnpm --filter '@rivet-dev/agentos-runtime-core...' build
           pnpm --filter @rivet-dev/agentos-vm-test-harness build
+          pnpm --filter @rivet-dev/agentos-test-harness build
           pnpm --filter @rivet-dev/agentos-core build
           pnpm --filter @rivet-dev/agentos build
       - name: Run all native sidecar tests through the selected VM backend
         run: cargo test --release -p agentos-native-sidecar --tests -- --test-threads=1
+      - name: Run artifact-backed V8-WASM and Wasmtime software parity
+        if: matrix.backend == 'wasmtime'
+        run: cargo test --release -p agentos-native-sidecar --test wasm_software_parity -- --ignored --nocapture --test-threads=1
       - name: Run owned pthread libc conformance
         if: matrix.backend == 'wasmtime'
         run: cargo test --release -p agentos-native-sidecar --test wasmtime_safety owned_pthread_libc_mutex_cond_tls_join_detach_and_cancel_conform -- --ignored --exact --nocapture --test-threads=1
@@ -424,6 +482,8 @@ jobs:
         run: cargo test -p agentos-client -- --test-threads=1
       - name: Run every registry software suite through the selected WASM backend
         run: pnpm exec turbo test --concurrency=1 --filter='@agentos-software/*'
+      - name: Run every registry software nightly suite through the selected WASM backend
+        run: pnpm exec turbo test:nightly --concurrency=1 --filter='@agentos-software/*'
       - name: Run deterministic actor, ACP, and agent-tool conformance
         run: pnpm --filter @rivet-dev/agentos test:e2e:run
       - name: Run mixed V8-JavaScript and Wasmtime resource-plateau smoke