Download dag and debug ipfs #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

bkontur merged 16 commits into main from bko-debug-ipfs

Jan 22, 2026

.github/workflows/integration-test.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -19,6 +19,7 @@ concurrency:
  
    env:

      NODE_VERSION: 22

      POLKADOT_SDK_VERSION: polkadot-stable2512

      POLKADOT_SDK_COMMIT: 81a3af9830ea8b6ff64b066b73b04bb3b675add5

      POLKADOT_SDK_BIN_DIR: ${{ github.workspace }}/.polkadot-sdk-bin

      ZOMBIENET_VERSION: v1.3.138

      ZOMBIENET_BIN_DIR: ${{ github.workspace }}/.zombienet-bin

    @@ -52,7 +53,7 @@ jobs:
  
            id: polkadot-sdk-cache

            with:

              path: ${{ env.POLKADOT_SDK_BIN_DIR }}

              key: polkadot-sdk-${{ env.POLKADOT_SDK_VERSION }}-binaries

              key: polkadot-sdk-${{ env.POLKADOT_SDK_VERSION }}-${{ env.POLKADOT_SDK_COMMIT }}-binaries

          # Download and extract binaries if cache missed

          - name: Download Polkadot SDK binaries

            if: steps.polkadot-sdk-cache.outputs.cache-hit != 'true'

    @@ -70,7 +71,7 @@ jobs:
  
              # TODO: remove when released: https://github.com/paritytech/polkadot-sdk/pull/10662

              git clone https://github.com/paritytech/polkadot-sdk.git

              cd polkadot-sdk

              git reset --hard b2bcb74b13f1a1e082f701e3e05ce1be44d16790

              git reset --hard $POLKADOT_SDK_COMMIT

              cargo build -p polkadot-omni-node -r

              cd ..

              cp polkadot-sdk/target/release/polkadot-omni-node .

    @@ -139,7 +140,7 @@ jobs:
  
            id: polkadot-sdk-cache

            with:

              path: ${{ env.POLKADOT_SDK_BIN_DIR }}

              key: polkadot-sdk-${{ env.POLKADOT_SDK_VERSION }}-binaries

              key: polkadot-sdk-${{ env.POLKADOT_SDK_VERSION }}-${{ env.POLKADOT_SDK_COMMIT }}-binaries

          - name: Cache Zombienet

            uses: actions/cache@v3

            id: zombienet-cache

    @@ -174,6 +175,8 @@ jobs:
  
            run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "smoldot"

          - name: Test store-chunked-data (Westend parachain)

            # TODO: rewrite to PAPI

            continue-on-error: true

            working-directory: examples

            run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"

    @@ -205,6 +208,8 @@ jobs:
  
            run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "smoldot"

          - name: Test store-chunked-data (Polkadot solochain)

            # TODO: rewrite to PAPI

            continue-on-error: true

            working-directory: examples

            run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"

Cargo.lock

Large diffs are not rendered by default.

Cargo.toml

Large diffs are not rendered by default.

examples/README.md

-Original file line number
+Diff line change
@@ Expand Up @@
     #### Example for Simple Authorizing and Store
-    ##### Using Legacy @polkadot/api (PJS)
-    ```
-    cd examples
-    npm install
-    node authorize_and_store.js
-    ```
     ##### Using Modern PAPI (Polkadot API)
     ```bash
@@ Expand Down @@

examples/authorize_and_store.js

This file was deleted.

examples/authorize_and_store_papi.js

-Original file line number
+Diff line change
@@ Expand Up / @@ -2,7 +2,7 @@ import assert from "assert"; @@
     import { createClient } from 'polkadot-api';
     import { getWsProvider } from 'polkadot-api/ws-provider';
     import { cryptoWaitReady } from '@polkadot/util-crypto';
-    import { authorizeAccount, fetchCid, store} from './api.js';
+    import { authorizeAccount, fetchCid, store, TX_MODE_FINALIZED_BLOCK } from './api.js';
     import { setupKeyringAndSigners } from './common.js';
     import { cidFromBytes } from "./cid_dag_metadata.js";
     import { bulletin } from './.papi/descriptors/dist/index.mjs';
@@ Expand Down Expand Up / @@ -39,6 +39,7 @@ async function main() { @@
                 whoAddress,
 ,
                 BigInt(100 * 1024 * 1024), // 100 MiB
+                TX_MODE_FINALIZED_BLOCK,
             );
             // Store data.
@@ Expand Down @@

examples/justfile

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -96,17 +96,17 @@ bulletin-solo-zombienet-start test_dir runtime="bulletin-polkadot-runtime":
  
        ROOT_DIR="{{ justfile_directory() }}/.."

        # Build the node binary (which will build the runtime as a dependency)

        cargo build --release -p polkadot-bulletin-chain

        POLKADOT_BULLETIN_BINARY_PATH="$ROOT_DIR/target/release/polkadot-bulletin-chain"

        cat <<EOF

        ⚡ Starting Bulletin chain with zombienet

            runtime   : {{ runtime }}

            test_dir  : {{ test_dir }}

            zombienet : $ZOMBIENET_BIN

            POLKADOT_BINARY_PATH : $POLKADOT_BINARY_PATH

            POLKADOT_PARACHAIN_BINARY_PATH : $POLKADOT_PARACHAIN_BINARY_PATH

            POLKADOT_BULLETIN_BINARY_PATH : $POLKADOT_BULLETIN_BINARY_PATH

        EOF

        cd "$ROOT_DIR"

        (yes y | POLKADOT_BULLETIN_BINARY_PATH="$ROOT_DIR/target/release/polkadot-bulletin-chain" \

        (yes y | POLKADOT_BULLETIN_BINARY_PATH="$POLKADOT_BULLETIN_BINARY_PATH" \

            $ZOMBIENET_BIN -p native spawn --dir {{ test_dir }} "$ROOT_DIR/zombienet/bulletin-polkadot-local.toml" > {{ test_dir }}/zombienet.log 2>&1) &

        cd -

    @@ -239,7 +239,7 @@ ipfs-start test_dir: _check-docker
  
        docker logs -f ipfs-node > {{ test_dir }}/ipfs-node.log 2>&1 &

        docker exec ipfs-node ipfs log level bitswap debug

        docker exec ipfs-node ipfs log level bitswap/client debug

        # Store container name in PID directory for cleanup

        echo "ipfs-node" > {{ test_dir }}/ipfs-docker.container

    @@ -285,6 +285,7 @@ ipfs-shutdown test_dir:
  
        echo "🛑 Shutting down IPFS Docker container..."

        docker stop ipfs-node 2>/dev/null || echo "   ⚠ Container not running or already stopped"

        docker rm ipfs-node 2>/dev/null || echo "   ⚠ Container not found or already removed"

        docker volume rm ipfs-data 2>/dev/null || echo "   ⚠ Volume not found or already removed"

        # Clean up container file if it exists

        [ -f "{{ test_dir }}/ipfs-docker.container" ] && rm "{{ test_dir }}/ipfs-docker.container" && echo "   ✓ Cleaned up container file" || true

examples/store_big_data.js

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -4,15 +4,16 @@ import fs from 'fs'
  
    import os from "os";

    import path from "path";

    import assert from "assert";

    import {authorizeAccount, store, } from "./api.js";

    import {cidFromBytes} from "./cid_dag_metadata.js";

    import { authorizeAccount, store, fetchCid, TX_MODE_FINALIZED_BLOCK } from "./api.js";

    import { buildUnixFSDagPB, cidFromBytes } from "./cid_dag_metadata.js";

    import {

        setupKeyringAndSigners,

        CHUNK_SIZE,

        HTTP_IPFS_API,

        newSigner,

        fileToDisk,

        filesAreEqual,

        generateTextImage

        generateTextImage,

    } from "./common.js";

    import { createClient } from 'polkadot-api';

    import { getWsProvider } from "polkadot-api/ws-provider";

    @@ -115,18 +116,6 @@ const ipfs = create({
  
        url: 'http://127.0.0.1:5001', // Local IPFS API

    });

    async function readFromIpfs(cid) {

        // Fetch the block (downloads via Bitswap if not local)

        console.log('Trying to get cid: ', cid);

        const chunks = [];

        for await (const chunk of ipfs.cat(cid)) {

            chunks.push(chunk);

        }

        const fullData = Buffer.concat(chunks);

        console.log('Received block: ', fullData);

        return fullData

    }

    // Optional signer discriminator, when we want to run the script in parallel and don't take care of nonces.

    // E.g.: node store_big_data.js --signer-disc=BB

    const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-disc="))?.split("=")[1] ?? null;

    @@ -142,6 +131,7 @@ async function main() {
  
            const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "bulletinimggen-"));

            const filePath = path.join(tmpDir, "image.jpeg");

            const downloadedFilePath = path.join(tmpDir, "downloaded.jpeg");

            const downloadedFileByDagPath = path.join(tmpDir, "downloadedByDag.jpeg");

            generateTextImage(filePath, "Hello, Bulletin big - " + new Date().toString(), "big");

            // Init WS PAPI client and typed api.

    @@ -166,6 +156,7 @@ async function main() {
  
                signers.map(a => a.address),

                100,

                BigInt(100 * 1024 * 1024), // 100 MiB

                TX_MODE_FINALIZED_BLOCK,

            );

            // Start 8 workers

    @@ -187,7 +178,22 @@ async function main() {
  
                throw new Error('❌ Storing chunks failed! Error:' + err.message);

            }

            console.log(`Storing DAG...`);

            let { rootCid, dagBytes } = await buildUnixFSDagPB(chunks, 0xb220);

            let cid = await store(bulletinAPI, signers[0].signer, dagBytes);

            console.log(`Downloading...${cid} / ${rootCid}`);

            let downloadedContent = await fetchCid(HTTP_IPFS_API, rootCid);

            console.log(`✅ Reconstructed file size: ${downloadedContent.length} bytes`);

            await fileToDisk(downloadedFileByDagPath, downloadedContent);

            filesAreEqual(filePath, downloadedFileByDagPath);

            assert.strictEqual(

                dataSize,

                downloadedContent.length,

                '❌ Failed to download all the data!'

            );

            // Check all chunks are there.

            console.log(`Downloading by chunks...`);

            let downloadedChunks = [];

            for (const chunk of chunks) {

                // Download the chunk from IPFS.

scripts/setup_parachain_prerequisites.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -32,7 +32,7 @@ echo " Checking out latest master..." @@
     # TODO:
     # git reset --hard origin/master
     # Let's use the same commit as Cargo.toml to avoid moving Polkadot-SDK
-    git reset --hard b2bcb74b13f1a1e082f701e3e05ce1be44d16790
+    git reset --hard 81a3af9830ea8b6ff64b066b73b04bb3b675add5
     # Build polkadot binary
     echo "   Building polkadot binary (this may take a while)..."
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Download dag and debug ipfs #184

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!