Chunked data example + nits

.github/workflows/integration-test.yml

@@ -72,3 +72,7 @@ jobs:
       - name: Run authorize and store (PAPI, RPC node)
         working-directory: examples
         run: just run-authorize-and-store "ws"
+
+      - name: Run store chunked data + DAG-PB (PJS-API, RPC node)
+        working-directory: examples
+        run: just run-store-chunked-data

examples/api.js

@@ -1,7 +1,5 @@
 import { cidFromBytes } from "./common.js";
 import { Binary } from '@polkadot-api/substrate-bindings';
-import * as multihash from "multiformats/hashes/digest";
-import {blake2AsU8a} from "@polkadot/util-crypto";
 
 export async function authorizeAccount(typedApi, sudoSigner, who, transactions, bytes) {
     console.log('Authorizing account...');
@@ -23,6 +21,7 @@ export async function store(typedApi, signer, data) {
     console.log('⬆️ Storing data with length=', data.length);
     const cid = await cidFromBytes(data);
 
+    // Convert data to Uint8Array then wrap in Binary for PAPI typed API
     const dataBytes = typeof data === 'string' ?
         new Uint8Array(Buffer.from(data)) :
         new Uint8Array(data);

examples/authorize_and_store_papi.js

@@ -51,7 +51,9 @@ async function main() {
             downloadedContent.toString(),
             '❌ dataToStore does not match downloadedContent!'
         );
-        console.log(`✅ Verified content - test passed!`);
+        console.log(`✅ Verified content!`);
+
+        console.log(`\n\n\n✅✅✅ Test passed! ✅✅✅`);
         resultCode = 0;
     } catch (error) {
         console.error("❌ Error:", error);

examples/authorize_and_store_papi_smoldot.js

@@ -94,7 +94,9 @@ async function main() {
             downloadedContent.toString(),
             '❌ dataToStore does not match downloadedContent!'
         );
-        console.log(`✅ Verified content - test passed!`);
+        console.log(`✅ Verified content!`);
+
+        console.log(`\n\n\n✅✅✅ Test passed! ✅✅✅`);
         resultCode = 0;
     } catch (error) {
         console.error("❌ Error:", error);

examples/common.js

@@ -3,6 +3,11 @@ import * as multihash from 'multiformats/hashes/digest'
 import { CID } from 'multiformats/cid'
 import { Keyring } from '@polkadot/keyring';
 import { getPolkadotSigner } from '@polkadot-api/signer';
+import * as dagPB from '@ipld/dag-pb'
+import { UnixFS } from 'ipfs-unixfs'
+import { createCanvas } from "canvas";
+import fs from "fs";
+import assert from "assert";
 
 export async function waitForNewBlock() {
     // TODO: wait for a new block.
@@ -64,3 +69,107 @@ export function setupKeyringAndSigners(sudoSeed, accountSeed) {
         whoAddress: whoAccount.address
     };
 }
+
+/**
+ * Build a UnixFS DAG-PB file node from raw chunks.
+ *
+ * (By default with SHA2 multihash)
+ */
+export async function buildUnixFSDagPB(chunks, mhCode = 0x12) {
+    if (!chunks?.length) {
+        throw new Error('❌ buildUnixFSDag: chunks[] is empty')
+    }
+
+    // UnixFS blockSizes = sizes of child blocks
+    const blockSizes = chunks.map(c => c.len)
+
+    console.log(`🧩 Building UnixFS DAG from chunks:
+  • totalChunks: ${chunks.length}
+  • blockSizes: ${blockSizes.join(', ')}`)
+
+    // Build UnixFS file metadata (no inline data here)
+    const fileData = new UnixFS({
+        type: 'file',
+        blockSizes
+    })
+
+    // DAG-PB node: our file with chunk links
+    const dagNode = dagPB.prepare({
+        Data: fileData.marshal(),
+        Links: chunks.map(c => ({
+            Name: '',
+            Tsize: c.len,
+            Hash: c.cid
+        }))
+    })
+
+    // Encode DAG-PB
+    const dagBytes = dagPB.encode(dagNode)
+
+    // Hash DAG to produce CIDv1
+    const rootCid = await cidFromBytes(dagBytes, dagPB.code, mhCode)
+
+    console.log(`✅ DAG root CID: ${rootCid.toString()}`)
+
+    return { rootCid, dagBytes }
+}
+
+/**
+ * Generates (dynamic) images based on the input text.
+ */
+export function generateTextImage(file, text, width = 800, height = 600) {
+    const canvas = createCanvas(width, height);
+    const ctx = canvas.getContext("2d");
+
+    // 🎨 Background
+    ctx.fillStyle = randomColor();
+    ctx.fillRect(0, 0, width, height);
+
+    // 🟠 Random shapes
+    for (let i = 0; i < 15; i++) {
+        ctx.beginPath();
+        ctx.fillStyle = randomColor();
+        ctx.arc(
+            Math.random() * width,
+            Math.random() * height,
+            Math.random() * 120,
+            0,
+            Math.PI * 2
+        );
+        ctx.fill();
+    }
+
+    // ✍️ Draw your text
+    ctx.font = "bold 40px Sans";
+    ctx.fillStyle = "white";
+    ctx.textAlign = "center";
+    ctx.textBaseline = "middle";
+
+    // Add text with shadow for readability
+    ctx.shadowColor = "black";
+    ctx.shadowBlur = 8;
+
+    ctx.fillText(text, width / 2, height / 2);
+
+    let jpegBytes = canvas.toBuffer("image/jpeg");
+    fs.writeFileSync(file, jpegBytes);
+    console.log("Saved to file:", file);
+}
+
+function randomColor() {
+    return `rgb(${rand255()}, ${rand255()}, ${rand255()})`;
+}
+
+function rand255() {
+    return Math.floor(Math.random() * 256);
+}
+
+export function filesAreEqual(path1, path2) {
+    const data1 = fs.readFileSync(path1);
+    const data2 = fs.readFileSync(path2);
+    assert.deepStrictEqual(data1.length, data2.length)
+
+    for (let i = 0; i < data1.length; i++) {
+        assert.deepStrictEqual(data1[i], data2[i])
+    }
+}

examples/justfile

@@ -260,3 +260,25 @@ run-authorize-and-store mode="ws": build npm-install
     just teardown-services    
     exit $EXAMPLE_EXIT
 
+# Run store chunked data example with Docker IPFS
+run-store-chunked-data: build npm-install
+    #!/usr/bin/env bash
+    set -e
+
+    echo "🚀 Starting store chunked data + DAG-PB workflow test ..."
+    echo ""
+
+    just setup-services
+    node store_chunked_data.js
+    EXAMPLE_EXIT=$?
+
+    echo ""
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    if [ $EXAMPLE_EXIT -eq 0 ]; then
+        echo "✅ Example completed successfully!"
+    else
+        echo "❌ Example failed with exit code $EXAMPLE_EXIT"
+    fi
+
+    just teardown-services
+    exit $EXAMPLE_EXIT

examples/package.json

@@ -11,10 +11,11 @@
     "@polkadot/keyring": "^13.5.8",
     "@polkadot/util": "^13.5.8",
     "@polkadot/util-crypto": "^13.5.8",
+    "canvas": "^3.2.0",
     "fs": "^0.0.1-security",
     "ipfs-http-client": "^60.0.1",
     "multiformats": "^13.4.1",
-    "polkadot-api": "^1.20.6",
+    "polkadot-api": "^1.22.0",
     "smoldot": "^2.0.40",
     "ws": "^8.18.0"
   },