Skip to content

Commit a94fa0b

Browse files
x3c41abkonturclaude
authored
CI improvement [2] (#193)
* WIP parametrisation * fix * fix timeout * Fix destructuring of store() return value in native_ipfs_dag_pb_chunked_data.js The store() function now returns { cid, blockHash, blockNumber } instead of just the CID. Update the destructuring to extract the cid property correctly. Co-Authored-By: Claude Opus 4.5 <[email protected]> * Add per-block transaction breakdown to storage statistics Replace the simple "Number of blocks" metric with a detailed breakdown showing the transaction count for each block in the measured range, including a visual bar chart for quick distribution analysis. Co-Authored-By: Claude Opus 4.5 <[email protected]> * Move start right before storing * Add start/end block * Let use 12->18 signers inparallel * Extract NUM_SIGNERS constant for parallel workers Replace hardcoded value with configurable constant set to 16. Co-Authored-By: Claude Opus 4.5 <[email protected]> * parametrised all the examples * unified JS logging and justfile echoing * moved logging into a separate file * revert * added missing parameter * fix(examples): calculate block stats from actual transaction blocks Calculate startBlock and endBlock from the first and last blocks where transactions were actually included, rather than querying the current block number at start/end times. Also show all blocks (including empty ones) in the TRANSACTIONS PER BLOCK section, displaying 0 txs for blocks without transactions. Co-Authored-By: Claude Opus 4.5 <[email protected]> * 32MiB -> 64MiB * fix(examples): use TX_MODE_FINALIZED_BLOCK for authorization calls Wait for finalized blocks in authorizePreimage and authorizeAccount calls to ensure authorization is complete before proceeding with store. Co-Authored-By: Claude Opus 4.5 <[email protected]> * fix big_data ipfs naming * renamed justfile too * added live westend and paseo setups in just, fixed bug in api.js (full size would not pass) * More nits (#209) * Increased tx pool from 20->40 MiB * WIP: try without `--network host` * WIP: big32 back * Wait finalized * Revert * Try 30MiB * WIP: wrong ip causes disconnects? * Fix docker ipfs ips * nit * revert back to host network with 127.0.0.1 * Run first * Why this is doing difference for Kubo? * Log connected peers * Disabled IPFS dht and discovery * less logs * WIP: temporary cache to speed to * wip * Removed `ws` ? * Fix listen-addr * nit * Better IPFS? * Nit * One more nit * omfg * Refactor DEFAULT_HTTP_IPFS_API and custom ports * Nit * Very nit * Refactor * Revert * Change pool * fmt + revert back to ws * Restart IPFS? * big96 * Parametrize image size * Stats * REvert * Removed comment * Last nits --------- Co-authored-by: Branislav Kontur <[email protected]> Co-authored-by: Claude Opus 4.5 <[email protected]>
1 parent 4a0c389 commit a94fa0b

16 files changed

+589
-187
lines changed

‎.github/workflows/integration-test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ jobs:
162162

163163
- name: Test store-big-data (Westend parachain)
164164
working-directory: examples
165-
run: just run-test-store-big-data "${{ env.TEST_DIR }}"
165+
run: just run-test-store-big-data "${{ env.TEST_DIR }}" "big32"
166166

167167
- name: Test authorize-preimage-and-store (Westend parachain)
168168
working-directory: examples
@@ -197,7 +197,7 @@ jobs:
197197

198198
- name: Test store-big-data (Polkadot solochain)
199199
working-directory: examples
200-
run: just run-test-store-big-data "${{ env.TEST_DIR }}"
200+
run: just run-test-store-big-data "${{ env.TEST_DIR }}" "big32"
201201

202202
- name: Test authorize-preimage-and-store (Polkadot solochain)
203203
working-directory: examples

‎examples/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ tar -xvzf kubo_v0.38.1_darwin-arm64.tar.gz
7979

8080
```shell
8181
docker pull ipfs/kubo:latest
82-
docker run -d --name ipfs-node -v ipfs-data:/data/ipfs -p 4001:4001 -p 8080:8080 -p 5001:5001 ipfs/kubo:latest
82+
docker run -d --name ipfs-node -v ipfs-data:/data/ipfs -p 4011:4011 -p 8283:8283 -p 5011:5011 ipfs/kubo:latest
8383
docker logs -f ipfs-node
8484
```
8585

‎examples/api.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export async function authorizeAccount(
4343
const accountTransactions = authValue.transactions;
4444
const accountBytes = authValue.bytes;
4545

46-
if (accountTransactions > transactions && accountBytes > bytes) {
46+
if (accountTransactions >= transactions && accountBytes >= bytes) {
4747
console.log('✅ Account authorization is sufficient.');
4848
continue;
4949
}
@@ -135,8 +135,8 @@ export async function store(typedApi, signer, data, cidCodec = null, mhCode = nu
135135
}
136136

137137
const tx = typedApi.tx.TransactionStorage.store({ data: toBinary(data) });
138-
await waitForTransaction(tx, signer, "Store", txMode, DEFAULT_TX_TIMEOUT_MS, client, txOpts);
139-
return expectedCid;
138+
const result = await waitForTransaction(tx, signer, "Store", txMode, DEFAULT_TX_TIMEOUT_MS, client, txOpts);
139+
return { cid: expectedCid, blockHash: result?.block?.hash, blockNumber: result?.block?.number };
140140
}
141141

142142
const UTILITY_BATCH_SIZE = 20;
@@ -254,7 +254,7 @@ export async function storeChunkedFile(typedApi, signer, filePath, chunkSize) {
254254
for (let i = 0; i < chunks.length; i++) {
255255
const { cid: expectedCid, bytes } = chunks[i];
256256
console.log(`📤 Storing chunk #${i + 1} CID: ${expectedCid}`);
257-
let cid = await store(typedApi, signer, bytes);
257+
let { cid } = await store(typedApi, signer, bytes);
258258
assert.deepStrictEqual(expectedCid, cid);
259259
console.log(`✅ Stored chunk #${i + 1} and CID equals!`);
260260
}

‎examples/authorize_and_store_papi.js‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,22 @@ import { createClient } from 'polkadot-api';
33
import { getWsProvider } from 'polkadot-api/ws-provider';
44
import { cryptoWaitReady } from '@polkadot/util-crypto';
55
import { authorizeAccount, fetchCid, store, TX_MODE_FINALIZED_BLOCK } from './api.js';
6-
import { setupKeyringAndSigners } from './common.js';
6+
import { setupKeyringAndSigners, DEFAULT_IPFS_GATEWAY_URL } from './common.js';
7+
import { logHeader, logConnection, logSuccess, logError, logTestResult } from './logger.js';
78
import { cidFromBytes } from "./cid_dag_metadata.js";
89
import { bulletin } from './.papi/descriptors/dist/index.mjs';
910

10-
// Command line arguments: [ws_url] [seed]
11+
// Command line arguments: [ws_url] [seed] [ipfs_api_url]
1112
const args = process.argv.slice(2);
1213
const NODE_WS = args[0] || 'ws://localhost:10000';
1314
const SEED = args[1] || '//Alice';
14-
const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
15+
const HTTP_IPFS_API = args[2] || DEFAULT_IPFS_GATEWAY_URL;
1516

1617
async function main() {
1718
await cryptoWaitReady();
1819

19-
console.log(`Connecting to: ${NODE_WS}`);
20-
console.log(`Using seed: ${SEED}`);
20+
logHeader('AUTHORIZE AND STORE TEST (WebSocket)');
21+
logConnection(NODE_WS, SEED, HTTP_IPFS_API);
2122

2223
let client, resultCode;
2324
try {
@@ -43,12 +44,12 @@ async function main() {
4344
);
4445

4546
// Store data.
46-
const cid = await store(bulletinAPI, whoSigner, dataToStore);
47-
console.log("✅ Data stored successfully with CID:", cid);
47+
const { cid } = await store(bulletinAPI, whoSigner, dataToStore);
48+
logSuccess(`Data stored successfully with CID: ${cid}`);
4849

4950
// Read back from IPFS
5051
let downloadedContent = await fetchCid(HTTP_IPFS_API, cid);
51-
console.log("✅ Downloaded content:", downloadedContent.toString());
52+
logSuccess(`Downloaded content: ${downloadedContent.toString()}`);
5253
assert.deepStrictEqual(
5354
cid,
5455
expectedCid,
@@ -59,12 +60,13 @@ async function main() {
5960
downloadedContent.toString(),
6061
'❌ dataToStore does not match downloadedContent!'
6162
);
62-
console.log(`✅ Verified content!`);
63+
logSuccess('Verified content!');
6364

64-
console.log(`\n\n\n✅✅✅ Test passed! ✅✅✅`);
65+
logTestResult(true, 'Authorize and Store Test');
6566
resultCode = 0;
6667
} catch (error) {
67-
console.error("❌ Error:", error);
68+
logError(`Error: ${error.message}`);
69+
console.error(error);
6870
resultCode = 1;
6971
} finally {
7072
if (client) client.destroy();

‎examples/authorize_and_store_papi_smoldot.js‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ import { createClient } from 'polkadot-api';
55
import { getSmProvider } from 'polkadot-api/sm-provider';
66
import { cryptoWaitReady } from '@polkadot/util-crypto';
77
import { authorizeAccount, fetchCid, store } from './api.js';
8-
import { setupKeyringAndSigners, waitForChainReady } from './common.js';
8+
import { setupKeyringAndSigners, waitForChainReady, DEFAULT_IPFS_GATEWAY_URL } from './common.js';
9+
import { logHeader, logConfig, logSuccess, logError, logTestResult } from './logger.js';
910
import { cidFromBytes } from "./cid_dag_metadata.js";
1011
import { bulletin } from './.papi/descriptors/dist/index.mjs';
1112

1213
// Constants
1314
// Increased sync time for parachain mode where smoldot needs more time to sync relay + para
1415
const SYNC_WAIT_SEC = 30;
1516
const SMOLDOT_LOG_LEVEL = 3; // 0=off, 1=error, 2=warn, 3=info, 4=debug, 5=trace
16-
const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
1717

1818
const TCP_BOOTNODE_REGEX = /^(\/ip[46]\/[^/]+)\/tcp\/(\d+)\/p2p\/(.+)$/;
1919
const WS_BOOTNODE_REGEX = /\/tcp\/\d+\/ws\/p2p\//;
@@ -95,19 +95,30 @@ async function createSmoldotClient(chainSpecPath, parachainSpecPath = null) {
9595

9696
async function main() {
9797
await cryptoWaitReady();
98-
98+
99+
logHeader('AUTHORIZE AND STORE TEST (Smoldot Light Client)');
100+
99101
// Get chainspec path from command line argument (required - main chain: relay for para, or solo)
100102
const chainSpecPath = process.argv[2];
101103
if (!chainSpecPath) {
102-
console.error('❌ Error: Chain spec path is required as first argument');
103-
console.error('Usage: node authorize_and_store_papi_smoldot.js <chain-spec-path> [parachain-spec-path]');
104-
console.error(' For parachains: <relay-chain-spec-path> <parachain-spec-path>');
105-
console.error(' For solochains: <solo-chain-spec-path>');
104+
logError('Chain spec path is required as first argument');
105+
console.error('Usage: node authorize_and_store_papi_smoldot.js <chain-spec-path> [parachain-spec-path] [ipfs-api-url]');
106+
console.error(' For parachains: <relay-chain-spec-path> <parachain-spec-path> [ipfs-api-url]');
107+
console.error(' For solochains: <solo-chain-spec-path> [ipfs-api-url]');
106108
process.exit(1);
107109
}
108-
110+
109111
// Optional parachain chainspec path (only needed for parachains)
110112
const parachainSpecPath = process.argv[3] || null;
113+
// Optional IPFS API URL
114+
const HTTP_IPFS_API = process.argv[4] || DEFAULT_IPFS_GATEWAY_URL;
115+
116+
logConfig({
117+
'Mode': 'Smoldot Light Client',
118+
'Chain Spec': chainSpecPath,
119+
'Parachain Spec': parachainSpecPath || 'N/A (solochain)',
120+
'IPFS API': HTTP_IPFS_API
121+
});
111122

112123
let sd, client, resultCode;
113124
try {
@@ -139,12 +150,12 @@ async function main() {
139150
);
140151

141152
// Store data.
142-
const cid = await store(bulletinAPI, whoSigner, dataToStore);
143-
console.log("✅ Data stored successfully with CID:", cid);
153+
const { cid } = await store(bulletinAPI, whoSigner, dataToStore);
154+
logSuccess(`Data stored successfully with CID: ${cid}`);
144155

145156
// Read back from IPFS
146157
let downloadedContent = await fetchCid(HTTP_IPFS_API, cid);
147-
console.log("✅ Downloaded content:", downloadedContent.toString());
158+
logSuccess(`Downloaded content: ${downloadedContent.toString()}`);
148159
assert.deepStrictEqual(
149160
cid,
150161
expectedCid,
@@ -155,12 +166,13 @@ async function main() {
155166
downloadedContent.toString(),
156167
'❌ dataToStore does not match downloadedContent!'
157168
);
158-
console.log(`✅ Verified content!`);
169+
logSuccess('Verified content!');
159170

160-
console.log(`\n\n\n✅✅✅ Test passed! ✅✅✅`);
171+
logTestResult(true, 'Authorize and Store Test (Smoldot)');
161172
resultCode = 0;
162173
} catch (error) {
163-
console.error("❌ Error:", error);
174+
logError(`Error: ${error.message}`);
175+
console.error(error);
164176
resultCode = 1;
165177
} finally {
166178
if (client) client.destroy();

‎examples/authorize_preimage_and_store_papi.js‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,17 @@ import assert from "assert";
22
import { createClient } from 'polkadot-api';
33
import { getWsProvider } from 'polkadot-api/ws-provider';
44
import { cryptoWaitReady } from '@polkadot/util-crypto';
5-
import { authorizeAccount, authorizePreimage, fetchCid, store, TX_MODE_IN_BLOCK } from './api.js';
6-
import { setupKeyringAndSigners, getContentHash } from './common.js';
5+
import { authorizeAccount, authorizePreimage, fetchCid, store, TX_MODE_IN_BLOCK, TX_MODE_FINALIZED_BLOCK } from './api.js';
6+
import { setupKeyringAndSigners, getContentHash, DEFAULT_IPFS_GATEWAY_URL } from './common.js';
7+
import { logHeader, logConnection, logSection, logSuccess, logError, logInfo, logTestResult } from './logger.js';
78
import { cidFromBytes } from "./cid_dag_metadata.js";
89
import { bulletin } from './.papi/descriptors/dist/index.mjs';
910

10-
const NODE_WS = 'ws://localhost:10000';
11-
const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
11+
// Command line arguments: [ws_url] [seed] [ipfs_api_url]
12+
const args = process.argv.slice(2);
13+
const NODE_WS = args[0] || 'ws://localhost:10000';
14+
const SEED = args[1] || '//Alice';
15+
const HTTP_IPFS_API = args[2] || DEFAULT_IPFS_GATEWAY_URL;
1216

1317
/**
1418
* Run a preimage authorization + store test.
@@ -23,7 +27,7 @@ const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
2327
* @param {object|null} client - Client for unsigned transactions
2428
*/
2529
async function runPreimageStoreTest(testName, bulletinAPI, sudoSigner, signer, signerAddress, cidCodec, mhCode, client) {
26-
console.log(`\n========== ${testName} ==========\n`);
30+
logSection(testName);
2731

2832
// Data to store
2933
const dataToStore = `Hello, Bulletin - ${testName} - ${new Date().toString()}`;
@@ -39,28 +43,30 @@ async function runPreimageStoreTest(testName, bulletinAPI, sudoSigner, signer, s
3943
bulletinAPI,
4044
sudoSigner,
4145
contentHash,
42-
BigInt(dataToStore.length)
46+
BigInt(dataToStore.length),
47+
TX_MODE_FINALIZED_BLOCK
4348
);
4449

4550
// If signer is provided, also authorize the account (to increment inc_providers/inc_sufficients for `CheckNonce`).
4651
if (signer != null && signerAddress != null) {
47-
console.log(`ℹ️ Also authorizing account ${signerAddress} to verify preimage auth is preferred`);
52+
logInfo(`Also authorizing account ${signerAddress} to verify preimage auth is preferred`);
4853
await authorizeAccount(
4954
bulletinAPI,
5055
sudoSigner,
5156
signerAddress,
5257
10, // dummy transactions
53-
BigInt(10000) // dummy bytes
58+
BigInt(10000), // dummy bytes
59+
TX_MODE_FINALIZED_BLOCK
5460
);
5561
}
5662

5763
// Store data
58-
const cid = await store(bulletinAPI, signer, dataToStore, cidCodec, mhCode, TX_MODE_IN_BLOCK, client);
59-
console.log("✅ Data stored successfully with CID:", cid.toString());
64+
const { cid } = await store(bulletinAPI, signer, dataToStore, cidCodec, mhCode, TX_MODE_IN_BLOCK, client);
65+
logSuccess(`Data stored successfully with CID: ${cid.toString()}`);
6066

6167
// Read back from IPFS
6268
const downloadedContent = await fetchCid(HTTP_IPFS_API, cid);
63-
console.log("✅ Downloaded content:", downloadedContent.toString());
69+
logSuccess(`Downloaded content: ${downloadedContent.toString()}`);
6470

6571
// Verify CID matches
6672
assert.deepStrictEqual(
@@ -76,20 +82,23 @@ async function runPreimageStoreTest(testName, bulletinAPI, sudoSigner, signer, s
7682
'❌ Stored data does not match downloaded content!'
7783
);
7884

79-
console.log(`✅ Verified content!`);
85+
logSuccess('Verified content!');
8086
}
8187

8288
async function main() {
8389
await cryptoWaitReady();
8490

91+
logHeader('AUTHORIZE PREIMAGE AND STORE TEST');
92+
logConnection(NODE_WS, SEED, HTTP_IPFS_API);
93+
8594
let client, resultCode;
8695
try {
8796
// Init WS PAPI client and typed api.
8897
client = createClient(getWsProvider(NODE_WS));
8998
const bulletinAPI = client.getTypedApi(bulletin);
9099

91100
// Signers.
92-
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Preimagesigner');
101+
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners(SEED, '//Preimagesigner');
93102

94103
// Test 1: Unsigned store with preimage auth (default CID config)
95104
await runPreimageStoreTest(
@@ -116,10 +125,11 @@ async function main() {
116125
client
117126
);
118127

119-
console.log(`\n\n\n✅✅✅ All tests passed! ✅✅✅`);
128+
logTestResult(true, 'Authorize Preimage and Store Test');
120129
resultCode = 0;
121130
} catch (error) {
122-
console.error("❌ Error:", error);
131+
logError(`Error: ${error.message}`);
132+
console.error(error);
123133
resultCode = 1;
124134
} finally {
125135
if (client) client.destroy();

‎examples/common.js‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ import fs from "fs";
66
import assert from "assert";
77

88
// ---- CONFIG ----
9-
export const HTTP_IPFS_API = 'http://127.0.0.1:8080'; // Local IPFS HTTP gateway
9+
export const DEFAULT_IPFS_API_URL = 'http://127.0.0.1:5011'; // IPFS HTTP API (for ipfs-http-client)
10+
export const DEFAULT_IPFS_GATEWAY_URL = 'http://127.0.0.1:8283'; // IPFS HTTP Gateway (for /ipfs/CID requests)
1011
export const CHUNK_SIZE = 1 * 1024 * 1024; // 1 MiB
1112
// -----------------
1213

@@ -47,7 +48,7 @@ export function newSigner(seed) {
4748
*
4849
* @param {string} file
4950
* @param {string} text
50-
* @param {"small" | "big"} size
51+
* @param {"small" | "big32" | "big64" | "big96"} size
5152
*/
5253
export function generateTextImage(file, text, size = "small") {
5354
console.log(`Generating ${size} image with text: ${text} to the file: ${file}...`);
@@ -60,14 +61,32 @@ export function generateTextImage(file, text, size = "small") {
6061
noise: 1,
6162
},
6263
// ~33 MiB
63-
big: {
64+
big32: {
6465
width: 6500,
6566
height: 5500,
6667
quality: 0.95,
6768
shapes: 1000,
6869
noise: 50,
6970
targetBytes: 32 * 1024 * 1024,
7071
},
72+
// ~64 MiB
73+
big64: {
74+
width: 7500,
75+
height: 5500,
76+
quality: 0.95,
77+
shapes: 1000,
78+
noise: 50,
79+
targetBytes: 65 * 1024 * 1024,
80+
},
81+
// ~96 MiB
82+
big96: {
83+
width: 9000,
84+
height: 6500,
85+
quality: 0.95,
86+
shapes: 1000,
87+
noise: 50,
88+
targetBytes: 98 * 1024 * 1024,
89+
},
7190
};
7291

7392
const cfg = presets[size];
@@ -108,7 +127,7 @@ export function generateTextImage(file, text, size = "small") {
108127

109128
// 🔧 Big images: tune quality to hit target size
110129
let imageBytes;
111-
if (size === "big" && cfg.targetBytes) {
130+
if ((size === "big32" || size === "big64" || size === "big96") && cfg.targetBytes) {
112131
let quality = cfg.quality;
113132

114133
do {
@@ -262,7 +281,7 @@ export function toHex(bytes) {
262281
// console.error("Unknown command:", command);
263282
// console.error("Usage:");
264283
// console.error(
265-
// ' node common.js generateTextImage "TEXT" [small|big]'
284+
// ' node common.js generateTextImage "TEXT" [small|big32|big64|big96]'
266285
// );
267286
// process.exit(1);
268287
// }

0 commit comments

Comments
 (0)