Consolidate store_big_data tests into single script

- Add --skip-authorize flag for live networks with pre-authorized accounts
- Add --skip-ipfs-verify flag to skip IPFS download verification
- Add --fast flag for TX_MODE_IN_POOL (broadcast only)
- Add --num-signers=N flag to configure parallel workers
- Remove redundant store_big_data_live.js
- Update justfile to use unified script with appropriate flags

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: x3c41a

examples/justfile

@@ -648,7 +648,7 @@ run-authorize-preimage-and-store-papi runtime ws_url="ws://localhost:10000" seed
 WESTEND_RPC := "wss://westend-bulletin-rpc.polkadot.io"
 PASEO_RPC := "wss://paseo-bulletin-rpc.polkadot.io"
 
-# Run store-big-data-live test against a live endpoint
+# Run store-big-data test against a live endpoint
 # Parameters:
 #   ws_url - WebSocket URL of the Bulletin chain node
 #   seed - Account seed phrase (must be pre-authorized on the network)
@@ -663,17 +663,19 @@ _run-live-tests ws_url seed ipfs_gateway_url="http://127.0.0.1:8283" image_size=
 
     just papi-generate "{{ ws_url }}"
 
+    # Live networks use pre-authorized accounts, so skip authorization
+    EXTRA_FLAGS="--skip-authorize"
+
     # Check if IPFS peers are configured for live network
-    EXTRA_FLAGS=""
     if [ -z "$WESTEND_PEER1_ID" ] || [ -z "$WESTEND_PEER1_ADDR" ]; then
         echo ""
         echo "⚠️  IPFS peer info not configured - running storage-only test."
         echo "   To enable IPFS verification, set WESTEND_PEER1_ID and WESTEND_PEER1_ADDR."
         echo ""
-        EXTRA_FLAGS="--skip-ipfs-verify"
+        EXTRA_FLAGS="$EXTRA_FLAGS --skip-ipfs-verify"
     fi
 
-    node store_big_data_live.js "{{ ws_url }}" "{{ seed }}" "{{ ipfs_gateway_url }}" "{{ image_size }}" $EXTRA_FLAGS
+    node store_big_data.js "{{ ws_url }}" "{{ seed }}" "{{ ipfs_gateway_url }}" "{{ image_size }}" $EXTRA_FLAGS
 
 # Run live tests against Westend Bulletin
 # Parameters:

examples/store_big_data.js

@@ -4,7 +4,7 @@ import fs from 'fs'
 import os from "os";
 import path from "path";
 import assert from "assert";
-import {authorizeAccount, store, fetchCid, TX_MODE_FINALIZED_BLOCK, TX_MODE_IN_BLOCK} from "./api.js";
+import {authorizeAccount, store, fetchCid, TX_MODE_FINALIZED_BLOCK, TX_MODE_IN_BLOCK, TX_MODE_IN_POOL} from "./api.js";
 import { buildUnixFSDagPB, cidFromBytes } from "./cid_dag_metadata.js";
 import {
     setupKeyringAndSigners,
@@ -30,6 +30,10 @@ import { bulletin } from './.papi/descriptors/dist/index.mjs';
 
 // Command line arguments: [ws_url] [seed] [ipfs_gateway_url] [image_size]
 // Note: --signer-disc=XX flag is also supported for parallel runs
+// Note: --num-signers=N flag controls number of parallel workers (default: 16)
+// Note: --skip-authorize flag skips account authorization (for live networks with pre-authorized accounts)
+// Note: --skip-ipfs-verify flag skips IPFS download verification
+// Note: --fast flag uses TX_MODE_IN_POOL for faster uploads (broadcast only)
 const args = process.argv.slice(2).filter(arg => !arg.startsWith('--'));
 const NODE_WS = args[0] || 'ws://localhost:10000';
 const SEED = args[1] || '//Alice';
@@ -38,7 +42,14 @@ const IPFS_GATEWAY_URL = args[2] || DEFAULT_IPFS_GATEWAY_URL;
 const IPFS_API_URL = IPFS_GATEWAY_URL.replace(':8283', ':5011');
 // Image size preset: small, big32, big64, big96
 const IMAGE_SIZE = args[3] || 'big64';
-const NUM_SIGNERS = 16;
+
+// Optional flags
+const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-disc="))?.split("=")[1] ?? null;
+const numSignersArg = process.argv.find(arg => arg.startsWith("--num-signers="))?.split("=")[1];
+const NUM_SIGNERS = numSignersArg ? parseInt(numSignersArg, 10) : 16;
+const SKIP_AUTHORIZE = process.argv.includes("--skip-authorize");
+const SKIP_IPFS_VERIFY = process.argv.includes("--skip-ipfs-verify");
+const FAST_MODE = process.argv.includes("--fast");
 
 // -------------------- queue --------------------
 const queue = [];
@@ -59,7 +70,7 @@ const stats = {
     blockHashes: {},   // Map block number -> block hash for timestamp lookups
 };
 
-function waitForQueueLength(targetLength, timeoutMs = 300000) {
+function waitForQueueLength(targetLength, timeoutMs = 600000) {
     return new Promise((resolve, reject) => {
         const start = Date.now();
 
@@ -101,15 +112,20 @@ async function processJob(typedApi, workerId, signer, chunk) {
         `Worker ${workerId} submitting tx for chunk ${chunk.cid} of size ${chunk.len} bytes`
     );
 
-    let { cid, blockHash, blockNumber } = await store(typedApi, signer.signer, chunk.bytes);
+    const txMode = FAST_MODE ? TX_MODE_IN_POOL : TX_MODE_IN_BLOCK;
+    let { cid, blockHash, blockNumber } = await store(typedApi, signer.signer, chunk.bytes, null, null, txMode);
     pushToResultQueue({ cid, blockNumber });
     if (blockNumber !== undefined) {
         stats.blockNumbers.push(blockNumber);
         if (blockHash && !stats.blockHashes[blockNumber]) {
             stats.blockHashes[blockNumber] = blockHash;
         }
     }
-    console.log(`Worker ${workerId} tx included in block #${blockNumber} with CID: ${cid}`);
+    if (FAST_MODE) {
+        console.log(`Worker ${workerId} tx broadcasted with CID: ${cid}`);
+    } else {
+        console.log(`Worker ${workerId} tx included in block #${blockNumber} with CID: ${cid}`);
+    }
 }
 
 // -------------------- helpers --------------------
@@ -181,6 +197,7 @@ async function printStatistics(dataSize, typedApi) {
     console.log(`│ File size           │ ${formatBytes(dataSize).padEnd(25)} │`);
     console.log(`│ Chunk/TX size       │ ${formatBytes(CHUNK_SIZE).padEnd(25)} │`);
     console.log(`│ Number of chunks    │ ${numTxs.toString().padEnd(25)} │`);
+    console.log(`│ Num signers         │ ${NUM_SIGNERS.toString().padEnd(25)} │`);
     console.log(`│ Avg txs per block   │ ${`${avgTxsPerBlock} (${numTxs}/${totalBlocksInRange})`.padEnd(25)} │`);
     console.log(`│ Time elapsed        │ ${formatDuration(elapsed).padEnd(25)} │`);
     console.log(`│ Blocks elapsed      │ ${`${blocksElapsed} (#${startBlock} → #${endBlock})`.padEnd(25)} │`);
@@ -230,19 +247,22 @@ export async function storeChunkedFile(api, filePath) {
 }
 
 // Connect to IPFS API (for ipfs-http-client operations like block.get)
-const ipfs = create({
-    url: IPFS_API_URL,
-});
-
-// Optional signer discriminator, when we want to run the script in parallel and don't take care of nonces.
-// E.g.: node store_big_data.js --signer-disc=BB
-const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-disc="))?.split("=")[1] ?? null;
+let ipfs = null;
+if (!SKIP_IPFS_VERIFY) {
+    ipfs = create({
+        url: IPFS_API_URL,
+    });
+}
 
 async function main() {
     await cryptoWaitReady()
 
     logHeader('STORE BIG DATA TEST');
-    logConnection(NODE_WS, SEED, IPFS_GATEWAY_URL);
+    logConnection(NODE_WS, SEED.length > 20 ? SEED.substring(0, 10) + '...' : SEED, IPFS_GATEWAY_URL);
+    console.log(`Image size: ${IMAGE_SIZE}, Num signers: ${NUM_SIGNERS}`);
+    if (SKIP_AUTHORIZE) console.log('Skipping authorization (--skip-authorize)');
+    if (SKIP_IPFS_VERIFY) console.log('Skipping IPFS verification (--skip-ipfs-verify)');
+    if (FAST_MODE) console.log('Fast mode enabled (--fast)');
 
     let client, resultCode;
     try {
@@ -255,29 +275,36 @@ async function main() {
         // Init WS PAPI client and typed api.
         client = createClient(getWsProvider(NODE_WS));
         const bulletinAPI = client.getTypedApi(bulletin);
-        const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
 
-        // Let's do parallelism with multiple accounts
+        // Create signers - use discriminator if provided for parallel runs
         const signers = Array.from({ length: NUM_SIGNERS }, (_, i) => {
-            if (!signerDiscriminator) {
-                return newSigner(`//Signer${i + 1}`)
-            } else {
+            if (signerDiscriminator) {
                 console.log(`Using signerDiscriminator: "//Signer${signerDiscriminator}${i + 1}"`);
-                return newSigner(`//Signer${signerDiscriminator}${i + 1}`)
+                return newSigner(`//Signer${signerDiscriminator}${i + 1}`);
+            } else {
+                return newSigner(`//Signer${i + 1}`);
             }
         });
 
-        // Authorize accounts.
-        await authorizeAccount(
-            bulletinAPI,
-            sudoSigner,
-            signers.map(a => a.address),
-            100,
-            BigInt(100 * 1024 * 1024), // 100 MiB
-            TX_MODE_FINALIZED_BLOCK,
-        );
+        console.log(`Using ${signers.length} signer(s) with addresses:`);
+        signers.forEach((s, i) => console.log(`  Signer ${i}: ${s.address}`));
+
+        // Authorize accounts (skip for live networks with pre-authorized accounts)
+        if (!SKIP_AUTHORIZE) {
+            const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
+            await authorizeAccount(
+                bulletinAPI,
+                sudoSigner,
+                signers.map(a => a.address),
+                100,
+                BigInt(100 * 1024 * 1024), // 100 MiB
+                TX_MODE_FINALIZED_BLOCK,
+            );
+        } else {
+            console.log('Skipping authorization (accounts must be pre-authorized on live networks)');
+        }
 
-        // Start 8 workers
+        // Start workers
         signers.forEach((signer, i) => {
             startWorker(bulletinAPI, i, signer);
         });
@@ -300,49 +327,59 @@ async function main() {
 
         console.log(`Storing DAG...`);
         let { rootCid, dagBytes } = await buildUnixFSDagPB(chunks, 0xb220);
+        const dagTxMode = FAST_MODE ? TX_MODE_IN_POOL : TX_MODE_IN_BLOCK;
         // Store with dag-pb codec (0x70) to match rootCid from buildUnixFSDagPB
         let { cid } = await store(
             bulletinAPI,
             signers[0].signer,
             dagBytes,
             0x70,   // dag-pb codec
             0xb220, // blake2b-256
-            TX_MODE_IN_BLOCK
+            dagTxMode
         );
-        console.log(`Downloading...${cid} / ${rootCid}`);
+        console.log(`DAG stored with CID: ${cid}`);
         assert.deepStrictEqual(cid, rootCid, '❌ CID mismatch between stored and computed DAG root');
-        let downloadedContent = await fetchCid(IPFS_GATEWAY_URL, rootCid);
-        console.log(`✅ Reconstructed file size: ${downloadedContent.length} bytes`);
-        await fileToDisk(downloadedFileByDagPath, downloadedContent);
-        filesAreEqual(filePath, downloadedFileByDagPath);
-        assert.strictEqual(
-            dataSize,
-            downloadedContent.length,
-            '❌ Failed to download all the data!'
-        );
-
-        // Check all chunks are there.
-        console.log(`Downloading by chunks...`);
-        let downloadedChunks = [];
-        for (const chunk of chunks) {
-            // Download the chunk from IPFS.
-            let block = await ipfs.block.get(chunk.cid, {timeout: 15000});
-            downloadedChunks.push(block);
-        }
-        let fullBuffer = Buffer.concat(downloadedChunks);
-        console.log(`✅ Reconstructed file size: ${fullBuffer.length} bytes`);
-        await fileToDisk(downloadedFilePath, fullBuffer);
-        filesAreEqual(filePath, downloadedFilePath);
-        assert.strictEqual(
-            dataSize,
-            fullBuffer.length,
-            '❌ Failed to download all the data!'
-        );
 
         // Print storage statistics
         await printStatistics(dataSize, bulletinAPI);
 
-        logTestResult(true, 'Store Big Data Test');
+        if (SKIP_IPFS_VERIFY) {
+            console.log('\n--skip-ipfs-verify flag set, skipping IPFS download verification');
+            console.log(`Root CID for manual verification: ${rootCid}`);
+            console.log(`IPFS Gateway URL: ${IPFS_GATEWAY_URL}/ipfs/${rootCid}`);
+            logTestResult(true, 'Store Big Data Test (Storage Only)');
+        } else {
+            console.log(`Downloading...${cid} / ${rootCid}`);
+            let downloadedContent = await fetchCid(IPFS_GATEWAY_URL, rootCid);
+            console.log(`✅ Reconstructed file size: ${downloadedContent.length} bytes`);
+            await fileToDisk(downloadedFileByDagPath, downloadedContent);
+            filesAreEqual(filePath, downloadedFileByDagPath);
+            assert.strictEqual(
+                dataSize,
+                downloadedContent.length,
+                '❌ Failed to download all the data!'
+            );
+
+            // Check all chunks are there.
+            console.log(`Downloading by chunks...`);
+            let downloadedChunks = [];
+            for (const chunk of chunks) {
+                // Download the chunk from IPFS.
+                let block = await ipfs.block.get(chunk.cid, {timeout: 15000});
+                downloadedChunks.push(block);
+            }
+            let fullBuffer = Buffer.concat(downloadedChunks);
+            console.log(`✅ Reconstructed file size: ${fullBuffer.length} bytes`);
+            await fileToDisk(downloadedFilePath, fullBuffer);
+            filesAreEqual(filePath, downloadedFilePath);
+            assert.strictEqual(
+                dataSize,
+                fullBuffer.length,
+                '❌ Failed to download all the data!'
+            );
+
+            logTestResult(true, 'Store Big Data Test');
+        }
         resultCode = 0;
     } catch (error) {
         logError(`Error: ${error.message}`);