test: add opt-in devnet scenarios

.github/workflows/ci_pull_request.yml

@@ -9,6 +9,7 @@ name: CI (Pull Request)
 
 on:
   pull_request:
+    types: [opened, reopened, synchronize, labeled, unlabeled]
   push:
     branches: ['main']
 
@@ -59,5 +60,6 @@ jobs:
     with:
       name: default
       profile: default
+      include_optional_scenarios: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-optional-scenarios') }}
       enable_reporting: false
     secrets: inherit

.github/workflows/ci_run.yml

@@ -20,6 +20,11 @@ on:
         description: 'Dependency profile declared in ci/dependency-profiles.json'
         required: true
         type: string
+      include_optional_scenarios:
+        description: 'Run optional extended scenarios after the core suite'
+        required: false
+        type: boolean
+        default: false
       enable_reporting:
         description: 'When true, file a GitHub issue with the scenario report'
         required: false
@@ -44,7 +49,7 @@ on:
 jobs:
   foc-start-test:
     runs-on: ["self-hosted", "linux", "x64", "4xlarge+disk"]
-    timeout-minutes: 100
+    timeout-minutes: 150
     permissions:
       contents: read
       issues: write
@@ -423,7 +428,12 @@ jobs:
         SKIP_REPORT_ON_PASS: ${{ inputs.skip_report_on_pass }}
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         SCENARIO_RUN_TYPE: ${{ inputs.name }}
-      run: python3 scenarios/run.py
+      run: |
+        if [[ "${{ inputs.include_optional_scenarios }}" == "true" ]]; then
+          python3 scenarios/run.py --include-optional
+        else
+          python3 scenarios/run.py
+        fi
 
     # Ensure scenario report exists even if tests didn't run (for issue reporting)
     - name: "EXEC: {Ensure scenario report exists}"

README_ADVANCED.md

@@ -1314,9 +1314,12 @@ Scenario tests are Python scripts that validate devnet state after startup. They
 ### Running scenarios
 
 ```bash
-# Run all scenarios
+# Run the core scenario suite
 python3 scenarios/run.py
 
+# Include extended optional scenarios
+python3 scenarios/run.py --include-optional
+
 # Run a single scenario directly
 python3 scenarios/test_basic_balances.py
 
@@ -1328,7 +1331,11 @@ Reports are written to `~/.foc-devnet/state/latest/scenario_report.md`.
 
 ### CI integration
 
-Scenarios run automatically in CI after the devnet starts. On nightly runs (or manual dispatch with `reporting` enabled), failures automatically create a GitHub issue with a full report.
+Scenarios run automatically in CI after the devnet starts. Pull requests run the
+core suite by default. Apply the `run-optional-scenarios` label to rerun the
+same pull-request CI with the extended scenarios included. On nightly runs (or
+manual dispatch with `reporting` enabled), failures automatically create a
+GitHub issue with a full report.
 
 CI resolves compatibility-sensitive dependencies from `ci/dependency-profiles.json`.
 Pull requests use the pinned `default` profile, while nightly `stability` runs use

scenarios/report.py

@@ -59,6 +59,11 @@ def get_version_info():
 ## Resolved dependencies
 $dependency_table
 
+## Scenario selection
+**$scenario_selection**
+
+$skipped_optional_tests
+
 ## Tests summary
 $test_summary
 """)
@@ -93,10 +98,25 @@ def _build_test_summary(results: list[TestResult]) -> str:
     return "\n\n".join(parts)
 
 
-def write_report(results: list[TestResult] | None = None, elapsed: int = 0):
+def _format_skipped_optional_tests(skipped_tests: list[str]) -> str:
+    if not skipped_tests:
+        return "No optional scenarios skipped."
+    return "Skipped optional scenarios:\n" + "\n".join(
+        f"- `{test_name}`" for test_name in skipped_tests
+    )
+
+
+def write_report(
+    results: list[TestResult] | None = None,
+    elapsed: int = 0,
+    selection: str = "core",
+    skipped_tests: list[str] | None = None,
+):
     """Write a markdown report to REPORT_MD. Returns path written."""
     if results is None:
         results = []
+    if skipped_tests is None:
+        skipped_tests = []
     total = len(results)
     passed = sum(1 for r in results if r.is_passed)
     content = _REPORT_TEMPLATE.substitute(
@@ -108,6 +128,8 @@ def write_report(results: list[TestResult] | None = None, elapsed: int = 0):
         ci_run_link=_build_ci_run_link(),
         version_info=f"```\n{get_version_info()}\n```",
         dependency_table=format_markdown_table(),
+        scenario_selection=selection,
+        skipped_optional_tests=_format_skipped_optional_tests(skipped_tests),
         test_summary=_build_test_summary(results),
     )
     with open(REPORT_MD, "w") as fh:

scenarios/run.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 """Scenario test runner — executes tests in order and generates a report.
 
-Run all tests:  python3 scenarios/run.py
-Run one test:   python3 scenarios/test_containers.py
+Run core tests:       python3 scenarios/run.py
+Run core + optional:  python3 scenarios/run.py --include-optional
+Run one test:         python3 scenarios/test_containers.py
 """
 
+import argparse
 import os
 import subprocess
 import sys
@@ -22,20 +24,36 @@
 from scenarios.report import TestResult, write_report
 
 # ── Scenario execution order ─────────────────────────────────
-# Each entry is (test_name, timeout_seconds)
+# Each entry is (test_name, timeout_seconds, optional). Optional scenarios are
+# discoverable and runnable directly, but omitted from the default core run.
 CREATE_DATASET_SMOKE_TIMEOUT_SECS = 1800
 
 ORDER = [
-    ("test_containers", 5),
-    ("test_basic_balances", 10),
+    ("test_containers", 5, False),
+    ("test_basic_balances", 10, False),
     # Allows setup plus five 280s Node attempts and retry delays.
-    ("test_create_dataset_smoke", CREATE_DATASET_SMOKE_TIMEOUT_SECS),
-    ("test_synapse_e2e", 600),
-    ("test_multi_copy_upload", 600),
-    ("test_caching_subsystem", 200),
+    ("test_create_dataset_smoke", CREATE_DATASET_SMOKE_TIMEOUT_SECS, False),
+    ("test_synapse_e2e", 600, False),
+    ("test_negative_permissions", 300, False),
+    ("test_multi_copy_upload", 600, False),
+    ("test_caching_subsystem", 200, False),
+    ("test_bulk_add", 1800, True),
+    ("test_termination_controls", 900, True),
 ]
 
 
+def select_scenarios(include_optional, order=ORDER):
+    """Return (selected, skipped) entries for an ordered scenario collection."""
+    selected = []
+    skipped = []
+    for scenario in order:
+        if scenario[2] and not include_optional:
+            skipped.append(scenario)
+        else:
+            selected.append(scenario)
+    return selected, skipped
+
+
 def _run_single_test(scenario_py_file, name, timeout_sec):
     """Run one scenario file as a subprocess, return a TestResult."""
     info(f"=== {name} (timeout: {timeout_sec}s) ===")
@@ -75,15 +93,26 @@ def _run_single_test(scenario_py_file, name, timeout_sec):
     )
 
 
-def run_tests():
-    """Run scenarios in ORDER. Returns list of TestResult."""
+def run_tests(include_optional=False):
+    """Run selected scenarios in ORDER. Returns list of TestResult."""
     pwd = os.path.dirname(os.path.abspath(__file__))
+    selected, _ = select_scenarios(include_optional)
     return [
         _run_single_test(os.path.join(pwd, f"{name}.py"), name, timeout)
-        for name, timeout in ORDER
+        for name, timeout, _ in selected
     ]
 
 
+def _parse_args():
+    parser = argparse.ArgumentParser(description="Run foc-devnet scenario tests")
+    parser.add_argument(
+        "--include-optional",
+        action="store_true",
+        help="run optional extended scenarios after the core scenarios",
+    )
+    return parser.parse_args()
+
+
 def _print_summary(results, elapsed):
     """Print a human-readable summary to stdout."""
     passed = sum(1 for r in results if r.is_passed)
@@ -116,10 +145,21 @@ def _print_ci_url():
 
 
 if __name__ == "__main__":
+    args = _parse_args()
+    _, skipped = select_scenarios(args.include_optional)
+    selection = "core + optional" if args.include_optional else "core"
+    info(f"Scenario selection: {selection}")
+    if skipped:
+        info(
+            "Skipping optional scenarios (use --include-optional): "
+            + ", ".join(name for name, _, _ in skipped)
+        )
     start = time.time()
-    results = run_tests()
+    results = run_tests(args.include_optional)
     elapsed = int(time.time() - start)
     _print_summary(results, elapsed)
-    print(f"Report: {write_report(results=results)}")
+    print(
+        f"Report: {write_report(results=results, selection=selection, skipped_tests=[name for name, _, _ in skipped])}"
+    )
     _print_ci_url()
     sys.exit(0 if all(r.is_passed for r in results) else 1)

scenarios/synapse-e2e/bulk-add.ts

@@ -0,0 +1,113 @@
+import assert from 'node:assert/strict'
+import * as SP from '@filoz/synapse-core/sp'
+import { findPieceIdsByCidCall, getActivePieceCount } from '@filoz/synapse-core/pdp-verifier'
+import { getPdpDataSet } from '@filoz/synapse-core/warm-storage'
+import { readContract } from 'viem/actions'
+import { createSynapse, prepareAccount, readAccountState } from './account.ts'
+import { freshMetadata, resolveEnvironment } from './environment.ts'
+import { fileSize, uploadFile } from './storage.ts'
+
+const REQUIRED_PIECES = 40
+const MAX_PIECES = 80
+const SMALL_PIECE_BYTES = 64 * 1024
+
+type PaymentSnapshot = {
+  pieceNumber: number
+  funds: bigint
+  availableFunds: bigint
+  lockupCurrent: bigint
+  lockupRate: bigint
+}
+
+function fixtureFor(pieceNumber: number): File {
+  const bytes = Buffer.alloc(SMALL_PIECE_BYTES, pieceNumber % 251)
+  bytes.write(`foc-devnet-bulk-add-${pieceNumber}`, 0, 'utf8')
+  return new File([bytes], `bulk-${pieceNumber.toString().padStart(3, '0')}.bin`, {
+    type: 'application/octet-stream',
+  })
+}
+
+async function main(): Promise<void> {
+  const environment = resolveEnvironment({ defaultUserIndex: 0, requireFiles: true })
+  if (environment.filePaths.length !== 1) throw new Error('bulk-add.ts accepts exactly one bootstrap file path')
+  const [bootstrapPath] = environment.filePaths
+  const synapse = createSynapse(environment)
+
+  // Prepare enough headroom for creation plus the mandatory 40 small additions.
+  await prepareAccount(synapse, (await fileSize(bootstrapPath)) + BigInt(SMALL_PIECE_BYTES * MAX_PIECES))
+  const { result } = await uploadFile(synapse, bootstrapPath, freshMetadata('bulk-add'), 1)
+  const copy = result.copies[0]
+  assert(copy != null, 'Expected a bootstrap data set')
+  const dataSet = await getPdpDataSet(synapse.client, { dataSetId: copy.dataSetId })
+  assert(dataSet != null && dataSet.live, `Bootstrap data set ${copy.dataSetId} is not live`)
+  const initialPieceCount = await getActivePieceCount(synapse.client, { dataSetId: copy.dataSetId })
+
+  const snapshots: PaymentSnapshot[] = []
+  const pieceCids = []
+  let replenishedAt: number | undefined
+  let previous = await readAccountState(synapse)
+
+  for (let pieceNumber = 1; pieceNumber <= MAX_PIECES; pieceNumber++) {
+    const added = await SP.upload(synapse.client, {
+      dataSetId: copy.dataSetId,
+      data: [fixtureFor(pieceNumber)],
+    })
+    assert.equal(added.pieces.length, 1, `Expected exactly one submitted piece at add ${pieceNumber}`)
+    const confirmed = await SP.waitForAddPieces({ statusUrl: added.statusUrl, timeout: 180_000 })
+    assert.equal(confirmed.piecesAdded, true, `Piece ${pieceNumber} was not added`)
+    assert.equal(confirmed.confirmedPieceIds.length, 1, `Piece ${pieceNumber} confirmation was incomplete`)
+
+    const current = await readAccountState(synapse)
+    snapshots.push({
+      pieceNumber,
+      funds: current.funds,
+      availableFunds: current.availableFunds,
+      lockupCurrent: current.lockupCurrent,
+      lockupRate: current.lockupRate,
+    })
+    pieceCids.push(added.pieces[0].pieceCid)
+    console.log(
+      `Added ${pieceNumber}: piece=${added.pieces[0].pieceCid} funds=${current.funds} ` +
+        `available=${current.availableFunds} lockup=${current.lockupCurrent} rate=${current.lockupRate}`
+    )
+
+    if (pieceNumber >= REQUIRED_PIECES && current.lockupCurrent > previous.lockupCurrent) {
+      replenishedAt = pieceNumber
+      break
+    }
+    previous = current
+  }
+
+  assert.equal(snapshots.length >= REQUIRED_PIECES, true, `Expected at least ${REQUIRED_PIECES} additions`)
+  assert(replenishedAt != null, `Lockup did not replenish within ${MAX_PIECES} additions`)
+  console.log(`Lockup replenished after piece ${replenishedAt}`)
+
+  const activePieceCount = await getActivePieceCount(synapse.client, { dataSetId: copy.dataSetId })
+  assert.equal(
+    activePieceCount,
+    initialPieceCount + BigInt(snapshots.length),
+    'On-chain active-piece count differs from successfully submitted pieces'
+  )
+  for (const pieceCid of pieceCids) {
+    const ids = await readContract(
+      synapse.client,
+      findPieceIdsByCidCall({
+        chain: synapse.client.chain,
+        dataSetId: copy.dataSetId,
+        pieceCid,
+        startPieceId: 0n,
+        limit: 2n,
+      })
+    )
+    assert.equal(ids.length, 1, `Submitted piece ${pieceCid} is not discoverable on-chain`)
+  }
+  console.log(
+    `=== SUCCESS: ${snapshots.length} distinct pieces are discoverable on data set ${copy.dataSetId}; ` +
+      `lockup replenished at ${replenishedAt} ===`
+  )
+}
+
+main().catch((error: unknown) => {
+  console.error(error)
+  process.exitCode = 1
+})