Sandbox Benchmark #65
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sandbox Benchmark | |
| on: | |
| pull_request: | |
| paths: | |
| - 'src/sandbox/**' | |
| - 'src/util/**' | |
| - 'src/run.ts' | |
| - 'src/merge-results.ts' | |
| - 'package.json' | |
| schedule: | |
| - cron: '0 0 * * 5' # Weekly on Friday at midnight UTC | |
| workflow_dispatch: | |
| inputs: | |
| iterations: | |
| description: 'Iterations per provider' | |
| required: false | |
| default: '100' | |
| concurrency: | |
| description: 'Concurrent sandboxes for burst/staggered tests' | |
| required: false | |
| default: '100' | |
| provider: | |
| description: 'Provider to run (leave empty for all)' | |
| required: false | |
| default: '' | |
| dry_run: | |
| description: 'Run without ingesting or committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| mode: | |
| description: 'Test mode (leave empty to run all)' | |
| required: false | |
| default: '' | |
| type: choice | |
| options: | |
| - '' | |
| - sequential | |
| - staggered | |
| - burst | |
| concurrency: | |
| group: benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| bench: | |
| name: Bench ${{ matrix.provider }} | |
| runs-on: namespace-profile-default | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: | |
| # - archil | |
| # - beam | |
| # - blaxel | |
| # - cloud-run | |
| # - cloudflare | |
| # - codesandbox | |
| # - collimate | |
| - createos | |
| # - daytona | |
| # - declaw | |
| # - e2b | |
| # - hopx | |
| # - isorun | |
| # - lelantos | |
| # - lightning | |
| # - modal | |
| # - namespace | |
| # - northflank | |
| # - quilt | |
| # - railway | |
| # - runloop | |
| # - superserve | |
| # - tenki | |
| # - tensorlake | |
| # - upstash | |
| # - vercel | |
| steps: | |
| - name: Print IP info | |
| run: curl -s https://ipinfo.io | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'npm' | |
| - name: Install dependencies | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| npm update | |
| else | |
| npm ci | |
| fi | |
| - name: Clear stale results from checkout | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider | |
| run: rm -rf results/ | |
| - name: Run benchmark | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider | |
| env: | |
| COMPUTESDK_API_KEY: ${{ secrets.COMPUTESDK_API_KEY }} | |
| ARCHIL_API_KEY: ${{ secrets.ARCHIL_API_KEY }} | |
| ARCHIL_REGION: ${{ secrets.ARCHIL_REGION }} | |
| ARCHIL_DISK_ID: ${{ secrets.ARCHIL_DISK_ID }} | |
| BEAM_TOKEN: ${{ secrets.BEAM_TOKEN }} | |
| BEAM_WORKSPACE_ID: ${{ secrets.BEAM_WORKSPACE_ID }} | |
| CLOUD_RUN_SANDBOX_URL: ${{ secrets.CLOUD_RUN_SANDBOX_URL }} | |
| CLOUD_RUN_SANDBOX_SECRET: ${{ secrets.CLOUD_RUN_SANDBOX_SECRET }} | |
| BL_API_KEY: ${{ secrets.BL_API_KEY }} | |
| BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} | |
| CLOUDFLARE_SANDBOX_URL: ${{ secrets.CLOUDFLARE_SANDBOX_URL }} | |
| CLOUDFLARE_SANDBOX_SECRET: ${{ secrets.CLOUDFLARE_SANDBOX_SECRET }} | |
| CSB_API_KEY: ${{ secrets.CSB_API_KEY }} | |
| # COLLIMATE_API_KEY: ${{ secrets.COLLIMATE_API_KEY }} | |
| CREATEOS_SANDBOX_API_KEY: ${{ secrets.CREATEOS_SANDBOX_API_KEY }} | |
| DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} | |
| DECLAW_API_KEY: ${{ secrets.DECLAW_API_KEY }} | |
| E2B_API_KEY: ${{ secrets.E2B_API_KEY }} | |
| HOPX_API_KEY: ${{ secrets.HOPX_API_KEY }} | |
| ISORUN_API_KEY: ${{ secrets.ISORUN_API_KEY }} | |
| # LELANTOS_API_KEY: ${{ secrets.LELANTOS_API_KEY }} | |
| LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} | |
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | |
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | |
| # NSC_TOKEN: ${{ secrets.NSC_TOKEN }} | |
| NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} | |
| NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }} | |
| # QUILT_API_KEY: ${{ secrets.QUILT_API_KEY }} | |
| # QUILT_BASE_URL: ${{ secrets.QUILT_BASE_URL }} | |
| # RAILWAY_API_TOKEN: ${{ secrets.RAILWAY_API_TOKEN }} | |
| # RAILWAY_ENVIRONMENT_ID: ${{ secrets.RAILWAY_ENVIRONMENT_ID }} | |
| RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} | |
| SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }} | |
| # TENKI_API_KEY: ${{ secrets.TENKI_API_KEY }} | |
| TENSORLAKE_API_KEY: ${{ secrets.TENSORLAKE_API_KEY }} | |
| UPSTASH_BOX_API_KEY: ${{ secrets.UPSTASH_BOX_API_KEY }} | |
| VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} | |
| VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} | |
| VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} | |
| run: | | |
| MODE_FLAG="" | |
| if [ -n "${{ github.event.inputs.mode }}" ]; then | |
| MODE_FLAG="--mode ${{ github.event.inputs.mode }}" | |
| fi | |
| npm run bench -- \ | |
| --provider ${{ matrix.provider }} \ | |
| --iterations ${{ github.event.inputs.iterations || (github.event_name == 'pull_request' && '10') || '100' }} \ | |
| --concurrency ${{ github.event.inputs.concurrency || (github.event_name == 'pull_request' && '10') || '100' }} \ | |
| $MODE_FLAG | |
| - name: Upload results | |
| if: always() && (github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider) | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.provider }} | |
| path: results/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| collect: | |
| name: Collect Results | |
| runs-on: namespace-profile-default | |
| needs: bench | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history so the rebase-on-push retry below has a merge base. | |
| # A shallow (depth-1) clone can make `git rebase origin/<branch>` | |
| # fail when the remote has advanced past the shallow boundary. | |
| fetch-depth: 0 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: | | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| npm update | |
| else | |
| npm ci | |
| fi | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| pattern: results-* | |
| - name: Merge results | |
| run: npx tsx src/merge-results.ts --input artifacts | |
| # - name: Ingest results to platform | |
| # if: github.event_name != 'pull_request' && github.event.inputs.dry_run != 'true' | |
| # continue-on-error: true | |
| # env: | |
| # INGEST_URL: ${{ secrets.INGEST_URL }} | |
| # INGEST_SECRET: ${{ secrets.INGEST_SECRET }} | |
| # run: npx tsx src/ingest.ts --type sandbox | |
| - run: npm run generate-svg | |
| - run: npm run generate-pricing-svg | |
| - name: Upload SVGs as artifacts | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sandbox-benchmark-svgs | |
| path: | | |
| results.svg | |
| *_tti.svg | |
| pricing.svg | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Post results to PR | |
| if: github.event_name == 'pull_request' | |
| continue-on-error: true | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const modes = [ | |
| { key: 'sequential_tti', label: 'Sequential' }, | |
| { key: 'staggered_tti', label: 'Staggered' }, | |
| { key: 'burst_tti', label: 'Burst' }, | |
| ]; | |
| let body = '## Sandbox Benchmark Results\n\n'; | |
| let hasResults = false; | |
| for (const mode of modes) { | |
| const latestPath = path.join('results', mode.key, 'latest.json'); | |
| if (!fs.existsSync(latestPath)) continue; | |
| const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); | |
| const results = data.results | |
| .filter(r => !r.skipped) | |
| .sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0)); | |
| if (results.length === 0) continue; | |
| hasResults = true; | |
| body += `### ${mode.label}\n\n`; | |
| body += '| # | Provider | Score | Median TTI | P95 | P99 | Status |\n'; | |
| body += '|---|----------|-------|------------|-----|-----|--------|\n'; | |
| results.forEach((r, i) => { | |
| const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--'; | |
| const median = (r.summary.ttiMs.median / 1000).toFixed(2) + 's'; | |
| const p95 = (r.summary.ttiMs.p95 / 1000).toFixed(2) + 's'; | |
| const p99 = (r.summary.ttiMs.p99 / 1000).toFixed(2) + 's'; | |
| const ok = r.iterations.filter(it => !it.error).length; | |
| const total = r.iterations.length; | |
| body += `| ${i + 1} | ${r.provider} | ${score} | ${median} | ${p95} | ${p99} | ${ok}/${total} |\n`; | |
| }); | |
| body += '\n'; | |
| } | |
| if (!hasResults) { | |
| body += '> No sandbox benchmark results were generated.\n\n'; | |
| } | |
| body += `---\n*[View full run](${runUrl}) · SVGs available as [build artifacts](${runUrl}#artifacts)*`; | |
| // Find and update existing comment or create new one | |
| const marker = '## Sandbox Benchmark Results'; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); | |
| } | |
| - name: Commit and push | |
| if: github.event_name != 'pull_request' && github.event.inputs.dry_run != 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add package.json package-lock.json results.svg *_tti.svg pricing.svg results/ | |
| git diff --cached --quiet && echo "No changes to commit" && exit 0 | |
| git commit -m "chore: update benchmark results [skip ci]" | |
| # Remote master can advance during the run (concurrent runs / other | |
| # commits), so a plain push fails non-fast-forward. Rebase onto the | |
| # latest remote and retry a few times before giving up. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| for attempt in 1 2 3 4 5; do | |
| # Rebase before each attempt (including the last) so every rebase | |
| # is followed by a push — otherwise the final iteration's rebase | |
| # would be wasted and a resolvable non-fast-forward could still fail. | |
| git fetch origin "${branch}" | |
| git rebase "origin/${branch}" || { git rebase --abort; exit 1; } | |
| if git push origin "HEAD:${branch}"; then | |
| echo "Pushed on attempt ${attempt}" | |
| exit 0 | |
| fi | |
| echo "Push rejected (attempt ${attempt}); will rebase and retry" | |
| done | |
| echo "Failed to push after multiple attempts" >&2 | |
| exit 1 |