Optimised CI

.github/workflows/integration-test.yml

@@ -155,68 +155,67 @@ jobs:
           echo "${ZOMBIENET_BIN_DIR}" >> "$GITHUB_PATH"
           echo "ZOMBIENET_BINARY=zombienet-linux-x64" >> "$GITHUB_ENV"
 
-      # Westend parachain
-      - name: Run authorize and store (PAPI, RPC node, Westend parachain)
+      # ========================================================================
+      # Westend parachain tests
+      # ========================================================================
+      - name: Start services (Westend parachain)
         working-directory: examples
         run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
+          TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
           echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-authorize-and-store "bulletin-westend-runtime" "ws"
-      - name: Run authorize and store (PAPI, smoldot, Westend parachain)
+          just start-services "$TEST_DIR" "bulletin-westend-runtime"
+
+      - name: Test authorize-and-store ws (Westend parachain)
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-authorize-and-store "bulletin-westend-runtime" "smoldot"
-      - name: Run store chunked data + DAG-PB (PJS-API, RPC node, Westend parachain)
+        run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "ws"
+
+      - name: Test authorize-and-store smoldot (Westend parachain)
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-store-chunked-data "bulletin-westend-runtime"
-      - name: Run store big data (PJS-API, RPC node, Westend parachain)
+        run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "smoldot"
+
+      - name: Test store-chunked-data (Westend parachain)
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> $GITHUB_ENV
-          mkdir -p "$TEST_DIR"
-          just run-store-big-data "bulletin-westend-runtime"
+        run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"
 
-#      TODO: Polkadot parachain
+      - name: Test store-big-data (Westend parachain)
+        working-directory: examples
+        run: just run-test-store-big-data "${{ env.TEST_DIR }}"
 
-      # Polkadot solochain
-      - name: Run authorize and store (PAPI, RPC node, Polkadot solochain)
+      - name: Stop services (Westend parachain)
+        if: always()
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-authorize-and-store "bulletin-polkadot-runtime" "ws"
-      - name: Run authorize and store (PAPI, smoldot, Polkadot solochain)
+        run: just stop-services "${{ env.TEST_DIR }}"
+
+      # ========================================================================
+      # Polkadot solochain tests
+      # ========================================================================
+      - name: Start services (Polkadot solochain)
         working-directory: examples
         run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
+          TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
           echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-authorize-and-store "bulletin-polkadot-runtime" "smoldot"
-      - name: Run store chunked data + DAG-PB (PJS-API, RPC node, Polkadot solochain)
+          just start-services "$TEST_DIR" "bulletin-polkadot-runtime"
+
+      - name: Test authorize-and-store ws (Polkadot solochain)
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
-          mkdir -p "$TEST_DIR"
-          just run-store-chunked-data "bulletin-polkadot-runtime"
+        run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "ws"
 
-      - name: Run store big data (PJS-API, RPC node, Polkadot solochain)
+      - name: Test authorize-and-store smoldot (Polkadot solochain)
         working-directory: examples
-        run: |
-          export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
-          echo "TEST_DIR=$TEST_DIR" >> $GITHUB_ENV
-          mkdir -p "$TEST_DIR"
-          just run-store-big-data "bulletin-polkadot-runtime"
+        run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "smoldot"
+
+      - name: Test store-chunked-data (Polkadot solochain)
+        working-directory: examples
+        run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"
+
+      - name: Test store-big-data (Polkadot solochain)
+        working-directory: examples
+        run: just run-test-store-big-data "${{ env.TEST_DIR }}"
+
+      - name: Stop services (Polkadot solochain)
+        if: always()
+        working-directory: examples
+        run: just stop-services "${{ env.TEST_DIR }}"
 
       # Collects logs from the last failed zombienet run.
       - name: Upload Zombienet logs (on failure)

examples/authorize_and_store_papi.js

@@ -7,20 +7,26 @@ import { setupKeyringAndSigners } from './common.js';
 import { cidFromBytes } from "./cid_dag_metadata.js";
 import { bulletin } from './.papi/descriptors/dist/index.mjs';
 
-const NODE_WS = 'ws://localhost:10000';
+// Command line arguments: [ws_url] [seed]
+const args = process.argv.slice(2);
+const NODE_WS = args[0] || 'ws://localhost:10000';
+const SEED = args[1] || '//Alice';
 const HTTP_IPFS_API = 'http://127.0.0.1:8080'   // Local IPFS HTTP gateway
 
 async function main() {
     await cryptoWaitReady();
-
+
+    console.log(`Connecting to: ${NODE_WS}`);
+    console.log(`Using seed: ${SEED}`);
+
     let client, resultCode;
     try {
         // Init WS PAPI client and typed api.
         client = createClient(getWsProvider(NODE_WS));
         const bulletinAPI = client.getTypedApi(bulletin);
 
         // Signers.
-        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
+        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners(SEED, '//Papisigner');
 
         // Data to store.
         const dataToStore = "Hello, Bulletin with PAPI - " + new Date().toString();
@@ -31,8 +37,8 @@ async function main() {
             bulletinAPI,
             sudoSigner,
             whoAddress,
-            1,
-            BigInt(dataToStore.length)
+            100,
+            BigInt(100 * 1024 * 1024), // 100 MiB
         );
 
         // Store data.

examples/authorize_and_store_papi_smoldot.js

@@ -10,7 +10,8 @@ import { cidFromBytes } from "./cid_dag_metadata.js";
 import { bulletin } from './.papi/descriptors/dist/index.mjs';
 
 // Constants
-const SYNC_WAIT_SEC = 15;
+// Increased sync time for parachain mode where smoldot needs more time to sync relay + para
+const SYNC_WAIT_SEC = 30;
 const SMOLDOT_LOG_LEVEL = 3; // 0=off, 1=error, 2=warn, 3=info, 4=debug, 5=trace
 const HTTP_IPFS_API = 'http://127.0.0.1:8080'   // Local IPFS HTTP gateway
 
@@ -120,8 +121,9 @@ async function main() {
         const bulletinAPI = client.getTypedApi(bulletin);
         await waitForChainReady(bulletinAPI);
 
-        // Signers.
-        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
+        // Signers: Use Bob for the account being authorized to avoid nonce conflicts
+        // when running after ws test (which uses Alice) on the same chain.
+        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Papismoldosigner');
 
         // Data to store.
         const dataToStore = "Hello, Bulletin with PAPI + Smoldot - " + new Date().toString();
@@ -132,8 +134,8 @@ async function main() {
             bulletinAPI,
             sudoSigner,
             whoAddress,
-            1,
-            BigInt(dataToStore.length)
+            100,
+            BigInt(100 * 1024 * 1024), // 100 MiB
         );
 
         // Store data.

examples/justfile

@@ -234,12 +234,12 @@ ipfs-start test_dir: _check-docker
     echo "   Container: ipfs-node"
     echo "   Waiting for container to start..."
     sleep 5
-
+    
     # Bitswap logging
     docker logs -f ipfs-node > {{ test_dir }}/ipfs-node.log 2>&1 &
     docker exec ipfs-node ipfs log level bitswap debug
     docker exec ipfs-node ipfs log level bitswap/client debug
-
+    
     # Store container name in PID directory for cleanup
     echo "ipfs-node" > {{ test_dir }}/ipfs-docker.container
 
@@ -316,12 +316,15 @@ ipfs-reconnect-start test_dir runtime:
     sleep 2
 
 # Generate PAPI descriptors
-papi-generate:
+# Generate PAPI descriptors from a running node
+# Parameters:
+#   ws_url - WebSocket URL of the node (default: ws://localhost:10000)
+papi-generate ws_url="ws://localhost:10000":
     #!/usr/bin/env bash
     set -e
 
-    echo "🔧 Generating PAPI descriptors..."
-    npm run papi:generate
+    echo "🔧 Generating PAPI descriptors from {{ ws_url }}..."
+    npx papi add -w "{{ ws_url }}" bulletin
 
 # Setup all services using Docker for IPFS
 # Parameters:
@@ -365,10 +368,82 @@ teardown-services test_dir:
     just ipfs-shutdown "{{ test_dir }}"
     echo "✅ Docker services stopped"
 
+# Start all services for a runtime (use with run-test-* recipes)
+# Usage: just start-services /tmp/my-test-dir bulletin-polkadot-runtime
+start-services test_dir runtime: npm-install
+    #!/usr/bin/env bash
+    set -e
+    mkdir -p "{{ test_dir }}"
+    just setup-services "{{ test_dir }}" "{{ runtime }}"
+
+# Stop all services
+# Usage: just stop-services /tmp/my-test-dir
+stop-services test_dir:
+    #!/usr/bin/env bash
+    echo "📁 Stopping services - test directory: {{ test_dir }}"
+    just teardown-services "{{ test_dir }}"
+    echo "✅ Services stopped"
+
+# Run authorize-and-store test only (services must already be running via start-services)
+# Parameters:
+#   test_dir - Test directory where services are running
+#   runtime - Runtime name for smoldot chainspec path resolution
+#   mode - Connection mode: "ws" (WebSocket RPC node) or "smoldot" (light client)
+run-test-authorize-and-store test_dir runtime mode="ws":
+    #!/usr/bin/env bash
+    set -e
+
+    if [ "{{ mode }}" = "smoldot" ]; then
+        echo "🧪 Running authorize and store test (mode: smoldot, runtime: {{ runtime }})..."
+        SCRIPT_NAME="authorize_and_store_papi_smoldot.js"
+    elif [ "{{ mode }}" = "ws" ]; then
+        echo "🧪 Running authorize and store test (mode: ws, runtime: {{ runtime }})..."
+        SCRIPT_NAME="authorize_and_store_papi.js"
+    else
+        echo "❌ Error: Invalid mode '{{ mode }}'. Must be 'ws' or 'smoldot'"
+        exit 1
+    fi
+
+    # Run the script with chainspec_path parameter only for smoldot mode
+    if [ "{{ mode }}" = "smoldot" ]; then
+        if [ "{{ runtime }}" = "bulletin-westend-runtime" ]; then
+            RELAY_CHAINSPEC_PATH="{{ test_dir }}/bob/cfg/westend-local.json"
+            PARACHAIN_CHAINSPEC_PATH="{{ test_dir }}/bulletin-westend-collator-2/cfg/westend-local-1006.json"
+            node $SCRIPT_NAME "$RELAY_CHAINSPEC_PATH" "$PARACHAIN_CHAINSPEC_PATH"
+        else
+            CHAINSPEC_PATH="{{ test_dir }}/bob/cfg/bulletin-polkadot-local.json"
+            node $SCRIPT_NAME "$CHAINSPEC_PATH"
+        fi
+    else
+        node $SCRIPT_NAME
+    fi
+
+# Run store-chunked-data test only (services must already be running via start-services)
+# Parameters:
+#   test_dir - Test directory where services are running
+run-test-store-chunked-data test_dir:
+    #!/usr/bin/env bash
+    set -e
+    echo "🧪 Running store chunked data test (test_dir: {{ test_dir }})..."
+    node store_chunked_data.js
+
+# Run store-big-data test only (services must already be running via start-services)
+# Parameters:
+#   test_dir - Test directory where services are running
+run-test-store-big-data test_dir:
+    #!/usr/bin/env bash
+    set -e
+    echo "🧪 Running store big data test (test_dir: {{ test_dir }})..."
+    node store_big_data.js
+
+# ============================================================================
+# Standalone recipes (with full setup/teardown) - kept for local dev convenience
+# ============================================================================
+
 # Run authorize and store example with Docker IPFS
 # Parameters:
 #   mode - Connection mode: "ws" (WebSocket RPC node) or "smoldot" (light client)
-#   runtime - Runtime name (e.g., "bulletin-polkadot-runtime", "bulletin-westend-runtime", "polkadot-bulletin-chain-runtime")
+#   runtime - Runtime name (e.g., "bulletin-polkadot-runtime", "bulletin-westend-runtime")
 run-authorize-and-store runtime mode="ws": npm-install
     #!/usr/bin/env bash
     set -e
@@ -467,3 +542,30 @@ run-store-big-data runtime: npm-install
 
     just teardown-services "$TEST_DIR"
     exit $EXAMPLE_EXIT
+
+# ============================================================================
+# Run tests against external environments
+# ============================================================================
+
+# Run authorize-and-store test against an external endpoint
+# Parameters:
+#   ws_url - WebSocket URL of the Bulletin chain node
+#   seed - Account seed phrase or dev seed (e.g., "//Alice" or "word1 word2 ...")
+run-tests-against ws_url seed: npm-install
+    #!/usr/bin/env bash
+    set -e
+    echo "🧪 Running authorize-and-store test against external endpoint..."
+    echo "   URL: {{ ws_url }}"
+    just papi-generate "{{ ws_url }}"
+
+    # TODO: connect to the local IPFS:8080
+    # TODO: find out the WS/IP/ live node address
+
+    # node authorize_and_store_papi.js "{{ ws_url }}" "{{ seed }}"
+    node store_big_data.js "{{ ws_url }}" "{{ seed }}"
+
+# Run authorize-and-store test against Westend
+# Parameters:
+#   seed - Account seed phrase or dev seed (e.g., "//Alice" or "word1 word2 ...")
+run-tests-against-westend seed:
+    just run-tests-against "wss://westend-bulletin-rpc.polkadot.io" "{{ seed }}"

examples/store_big_data.js

@@ -18,9 +18,11 @@ import { createClient } from 'polkadot-api';
 import { getWsProvider } from "polkadot-api/ws-provider";
 import { bulletin } from './.papi/descriptors/dist/index.mjs';
 
-// ---- CONFIG ----
-const NODE_WS = 'ws://localhost:10000';
-// ----
+// Command line arguments: [ws_url] [seed]
+// Note: --signer-disc=XX flag is also supported for parallel runs
+const args = process.argv.slice(2).filter(arg => !arg.startsWith('--'));
+const NODE_WS = args[0] || 'ws://localhost:10000';
+const SEED = args[1] || '//Alice';
 
 // -------------------- queue --------------------
 const queue = [];
@@ -132,17 +134,20 @@ const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-di
 async function main() {
     await cryptoWaitReady()
 
+    console.log(`Connecting to: ${NODE_WS}`);
+    console.log(`Using seed: ${SEED}`);
+
     let client, resultCode;
     try {
         const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "bulletinimggen-"));
         const filePath = path.join(tmpDir, "image.jpeg");
         const downloadedFilePath = path.join(tmpDir, "downloaded.jpeg");
-        generateTextImage(filePath, "Hello, Bulletin - " + new Date().toString(), "big");
+        generateTextImage(filePath, "Hello, Bulletin big - " + new Date().toString(), "big");
 
         // Init WS PAPI client and typed api.
         client = createClient(getWsProvider(NODE_WS));
         const bulletinAPI = client.getTypedApi(bulletin);
-        const { sudoSigner, _ } = setupKeyringAndSigners('//Alice', '//Alice');
+        const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
 
         // Let's do parallelism with multiple accounts
         const signers = Array.from({ length: 12 }, (_, i) => {

examples/store_chunked_data.js

@@ -214,12 +214,12 @@ async function main() {
             fs.unlinkSync(FILE_PATH);
             console.log(`File ${FILE_PATH} removed.`);
         }
-        generateTextImage(FILE_PATH, "Hello, Bulletin with PAPI - " + new Date().toString(), "small");
+        generateTextImage(FILE_PATH, "Hello, Bulletin with PAPI chunked - " + new Date().toString(), "small");
 
         // Init WS PAPI client and typed api.
         client = createClient(getWsProvider(NODE_WS));
         const bulletinAPI = client.getTypedApi(bulletin);
-        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
+        const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Chunkedsigner');
 
         // Authorize an account.
         await authorizeAccount(
@@ -239,7 +239,7 @@ async function main() {
         console.log('✅ Connected to Bulletin node')
 
         const keyring = new Keyring({ type: 'sr25519' })
-        const pair = keyring.addFromUri('//Alice')
+        const pair = keyring.addFromUri('//Chunkedsigner')
         const sudoPair = keyring.addFromUri('//Alice')
         let { nonce } = await api.query.system.account(pair.address);
         const nonceMgr = new NonceManager(nonce);