Skip to content

Commit 08527d6

Browse files
ftheirsftheirs
committed
improve upload method
1 parent 5951ebc commit 08527d6

File tree

3 files changed

+255
-43
lines changed

3 files changed

+255
-43
lines changed

‎src/files.ts‎

Lines changed: 71 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import {
2-
type StorageHubClient,
2+
StorageHubClient,
33
filesystemAbi,
44
FileMetadata,
55
FileTrie,
@@ -9,23 +9,25 @@ import {
99
ReplicationLevel,
1010
type EvmWriteOptions,
1111
} from "@storagehub-sdk/core";
12-
import type {
13-
MspClient,
14-
StorageFileInfo,
15-
UploadReceipt,
16-
UploadOptions,
17-
} from "@storagehub-sdk/msp-client";
12+
import { MspClient } from "@storagehub-sdk/msp-client";
13+
import type { StorageFileInfo, UploadReceipt, UploadOptions } from "@storagehub-sdk/msp-client";
1814
import { randomFillSync } from "node:crypto";
1915
import { createReadStream, createWriteStream } from "node:fs";
2016
import { mkdir, stat } from "node:fs/promises";
21-
import { dirname } from "node:path";
17+
import { dirname, resolve } from "node:path";
2218
import { Readable } from "node:stream";
2319
import { parseEventLogs } from "viem";
2420
import type { Hex, Log, PublicClient, RpcLog } from "viem";
21+
import type { ApiPromise } from "@polkadot/api";
2522
import { getLogger } from "./log.js";
2623
import type { PolkadotApi } from "./types.js";
2724
import { DEFAULT_BLOCK_TIME_MS } from "./buckets.js";
2825
import { sleep } from "./helpers/utils.js";
26+
import { readNumberEnv } from "./helpers/validation.js";
27+
import {
28+
waitForFinalizedAtLeast,
29+
waitForStorageRequestExistsOnChain,
30+
} from "./userApi.js";
2931

3032
// Re-export SDK types for convenience
3133
export type { FileInfo, UploadReceipt, UploadOptions };
@@ -461,6 +463,67 @@ export async function waitForStorageRequest(
461463
return { receipt, blockNumber };
462464
}
463465

466+
export type WaitForStorageRequestReadyForUploadParams = Readonly<{
467+
/**
468+
* Base SR confirmation (EVM receipt + StorageRequestIssued event).
469+
*/
470+
publicClient: PublicClient;
471+
txHash: Hex;
472+
fileKey: `0x${string}`;
473+
bucketId: `0x${string}`;
474+
location: string;
475+
filesystemContractAddress: `0x${string}`;
476+
expectedWho: `0x${string}`;
477+
/**
478+
* Follow-up readiness checks.
479+
*/
480+
userApi: ApiPromise;
481+
mspClient: MspClient;
482+
}>;
483+
484+
/**
485+
* Wait for SR to be "ready for upload":
486+
* - EVM receipt + StorageRequestIssued event
487+
* - Substrate storage request exists
488+
* - Finalized lag (indexers/backend ingest)
489+
* - MSP expects the fileKey (will accept upload)
490+
*/
491+
export async function waitForStorageRequestReadyForUpload(
492+
params: WaitForStorageRequestReadyForUploadParams
493+
): Promise<WaitForStorageRequestResult> {
494+
const sr = await waitForStorageRequest({
495+
publicClient: params.publicClient,
496+
txHash: params.txHash,
497+
fileKey: params.fileKey,
498+
bucketId: params.bucketId,
499+
location: params.location,
500+
filesystemContractAddress: params.filesystemContractAddress,
501+
expectedWho: params.expectedWho,
502+
});
503+
504+
await waitForStorageRequestExistsOnChain(params.userApi, params.fileKey, {
505+
timeoutMs: 120_000,
506+
intervalMs: 3_000,
507+
});
508+
509+
// Backends often react to finalized state; wait a small finalized lag after SR block.
510+
if (typeof sr.blockNumber === "bigint") {
511+
const lagBlocks = BigInt(readNumberEnv("SR_FINALIZATION_LAG_BLOCKS", 1));
512+
const target = sr.blockNumber + lagBlocks;
513+
await waitForFinalizedAtLeast(params.userApi, target, 120_000);
514+
}
515+
516+
await waitForMspToExpectFileKey({
517+
mspClient: params.mspClient,
518+
bucketId: params.bucketId,
519+
fileKey: params.fileKey,
520+
timeoutMs: readNumberEnv("MSP_EXPECT_FILEKEY_TIMEOUT_MS", 120_000),
521+
intervalMs: readNumberEnv("MSP_EXPECT_FILEKEY_POLL_MS", 3_000),
522+
});
523+
524+
return sr;
525+
}
526+
464527
/**
465528
* Issue a storage request for a file.
466529
* This submits an EVM transaction to request storage from an MSP.
@@ -793,38 +856,3 @@ export async function waitForMspFileStatus(
793856
);
794857
}
795858

796-
/**
797-
* Generate a test file with random content.
798-
* @param sizeBytes Size of the file in bytes
799-
* @param prefix Optional prefix for the content
800-
* @returns Buffer with the file content
801-
*/
802-
export function generateTestFile(
803-
sizeBytes: number,
804-
prefix = "test-file"
805-
): Buffer {
806-
const content = Buffer.alloc(sizeBytes);
807-
const prefixBytes = Buffer.from(`${prefix}-`);
808-
809-
// Write prefix at the start
810-
prefixBytes.copy(content, 0);
811-
812-
// Fill the rest with random data or pattern
813-
for (let i = prefixBytes.length; i < sizeBytes; i++) {
814-
// Use a simple pattern for reproducibility
815-
content[i] = i % 256;
816-
}
817-
818-
return content;
819-
}
820-
821-
/**
822-
* Generate a unique file path for testing.
823-
* @param prefix Optional prefix for the file name
824-
* @returns A unique file path like "test-20260106-143025.bin"
825-
*/
826-
export function generateTestFilePath(prefix = "test"): string {
827-
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
828-
const random = Math.random().toString(36).substring(2, 8);
829-
return `${prefix}-${timestamp}-${random}.bin`;
830-
}

‎src/processors/files.ts‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
import { StorageHubClient, ReplicationLevel } from "@storagehub-sdk/core";
2+
import { MspClient, type Session } from "@storagehub-sdk/msp-client";
3+
import { createPublicClient, http } from "viem";
4+
import { privateKeyToAccount } from "viem/accounts";
5+
import { resolve } from "node:path";
6+
import { readEnv } from "../config.js";
7+
import { DEFAULT_EVM_RPC_TIMEOUT_MS } from "../config/constants.js";
8+
import {
9+
ensureVars,
10+
persistVars,
11+
requirePersistedVar,
12+
requirePersistedVarNumber,
13+
requirePersistedVarString,
14+
requireVarString,
15+
type ArtilleryContext,
16+
type ArtilleryEvents,
17+
type Done,
18+
} from "../helpers/artillery.js";
19+
import { toError } from "../helpers/errors.js";
20+
import { createEmitter } from "../helpers/metrics.js";
21+
import { ensure0xPrefix } from "../helpers/validation.js";
22+
import { NETWORKS } from "../networks.js";
23+
import { pickSequentialResource } from "../resources/index.js";
24+
import { buildMspHttpClientConfig } from "../sdk/mspHttpConfig.js";
25+
import { createViemWallet, toViemChain } from "../sdk/viemWallet.js";
26+
import {
27+
computeFileKeyFromMetadata,
28+
filePathToWebStream,
29+
waitForMspFileStatus,
30+
waitForStorageRequestReadyForUpload,
31+
} from "../files.js";
32+
import {
33+
getUserApiSingleton,
34+
waitForStorageRequestFulfilledFinalized,
35+
} from "../userApi.js";
36+
37+
// Default bucket location used by our tests.
38+
const REMOTE_LOCATION = "/";
39+
40+
/**
41+
* Condensed upload flow:
42+
* - pick resource (deterministic)
43+
* - compute fileKey
44+
* - issue storage request + waits
45+
* - upload + waits
46+
*
47+
* Leaves all required vars persisted into `context.vars` for downstream steps (e.g. delete).
48+
*/
49+
export async function uploadFileFlow(
50+
context: ArtilleryContext,
51+
events: ArtilleryEvents,
52+
done?: Done
53+
): Promise<void> {
54+
const m = createEmitter(context, events);
55+
56+
try {
57+
const vars = ensureVars(context);
58+
const bucketId = requirePersistedVarString(context, "__uploadBucketId");
59+
const pk = ensure0xPrefix(requireVarString(vars, "privateKey"), 32);
60+
const account = privateKeyToAccount(pk);
61+
const sequence = requirePersistedVarNumber(context, "__accountIndexRaw");
62+
const session = requirePersistedVar(context, "__siweSession") as Session;
63+
64+
const picked = pickSequentialResource({ sequence });
65+
const localFilePath = resolve(process.cwd(), picked.path);
66+
const sizeBytes = picked.sizeBytes;
67+
const fingerprint = picked.fingerprint;
68+
69+
const fileKey = await computeFileKeyFromMetadata({
70+
owner: account.address,
71+
bucketId,
72+
location: REMOTE_LOCATION,
73+
size: BigInt(sizeBytes),
74+
fingerprint,
75+
});
76+
77+
const env = readEnv();
78+
const network = NETWORKS[env.network];
79+
const { chain, transportUrl } = toViemChain(network);
80+
81+
const publicClient = createPublicClient({
82+
chain,
83+
transport: http(transportUrl, { timeout: DEFAULT_EVM_RPC_TIMEOUT_MS }),
84+
});
85+
86+
const config = buildMspHttpClientConfig(network);
87+
const mspClient = await MspClient.connect(config, async () => session);
88+
const info = await mspClient.info.getInfo();
89+
const mspId = ensure0xPrefix(info.mspId, 32);
90+
const peerIds: string[] = info.multiaddresses
91+
.map((addr) => {
92+
const parts = addr.split("/");
93+
return parts[parts.length - 1] || "";
94+
})
95+
.filter((id) => id.length > 0);
96+
97+
const walletClient = createViemWallet(network, account);
98+
const storageHubClient = new StorageHubClient({
99+
rpcUrl: transportUrl,
100+
chain,
101+
walletClient,
102+
filesystemContractAddress: network.chain.filesystemPrecompileAddress,
103+
});
104+
105+
const srStart = Date.now();
106+
const txHash = await storageHubClient.issueStorageRequest(
107+
bucketId,
108+
REMOTE_LOCATION,
109+
fingerprint,
110+
BigInt(sizeBytes),
111+
mspId,
112+
peerIds,
113+
ReplicationLevel.Basic,
114+
1
115+
);
116+
if (!txHash) throw new Error("issueStorageRequest returned undefined txHash");
117+
m.histogram("uploadFlow.issueStorageRequest.ms", Date.now() - srStart);
118+
119+
// Wait until StorageRequest is processed and the backend is expecting the filekey
120+
const userApi = await getUserApiSingleton(network);
121+
const readyStart = Date.now();
122+
await waitForStorageRequestReadyForUpload({
123+
publicClient,
124+
txHash,
125+
fileKey,
126+
bucketId,
127+
location: REMOTE_LOCATION,
128+
filesystemContractAddress: network.chain.filesystemPrecompileAddress,
129+
expectedWho: account.address,
130+
userApi,
131+
mspClient,
132+
});
133+
m.histogram("uploadFlow.wait.readyForUpload.ms", Date.now() - readyStart);
134+
135+
// Upload file bytes to MSP
136+
const upStart = Date.now();
137+
const uploadReceipt = await mspClient.files.uploadFile(
138+
bucketId,
139+
fileKey,
140+
filePathToWebStream(localFilePath),
141+
account.address,
142+
REMOTE_LOCATION,
143+
{
144+
mspDistribution: true,
145+
contentLength: sizeBytes,
146+
}
147+
);
148+
m.histogram("uploadFlow.uploadFile.ms", Date.now() - upStart);
149+
150+
// Wait after upload (chain fulfillment + MSP ready)
151+
const fulfillStart = Date.now();
152+
await waitForStorageRequestFulfilledFinalized(userApi, fileKey);
153+
m.histogram("uploadFlow.wait.fulfilledChain.ms", Date.now() - fulfillStart);
154+
155+
const mspReadyStart = Date.now();
156+
await waitForMspFileStatus({
157+
mspClient,
158+
bucketId,
159+
fileKey,
160+
desiredStatus: "ready",
161+
timeoutMs: 660_000,
162+
intervalMs: 3_000,
163+
});
164+
m.histogram("uploadFlow.wait.mspReady.ms", Date.now() - mspReadyStart);
165+
166+
// Persist for follow-up steps (delete / waits).
167+
persistVars(context, {
168+
__uploadFileKey: fileKey,
169+
__uploadStorageRequestTxHash: txHash,
170+
__uploadLocation: REMOTE_LOCATION,
171+
__uploadLocalFilePath: localFilePath,
172+
__uploadFileSizeBytes: sizeBytes,
173+
__uploadFingerprint: fingerprint,
174+
__uploadOwner: account.address,
175+
});
176+
177+
void uploadReceipt;
178+
done?.();
179+
} catch (err) {
180+
done?.(toError(err));
181+
}
182+
}
183+

‎src/processors/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,4 @@ export * from "./metrics.js";
1616
export * from "./msp-unauth.js";
1717
export * from "./siwe-bootstrap.js";
1818
export * from "./examples.js";
19+
export * from "./files.js";

0 commit comments

Comments
 (0)