perf(sftp): reduce checksum CPU with digest chains

ui/koko/composables/sftp/useSftpTransferEndpoint.ts

@@ -164,6 +164,7 @@ export function useSftpTransferEndpoint(
         path: input.targetPath,
         size: input.totalBytes,
         sha256: input.sha256,
+        chunk_size: 2 * 1024 * 1024,
         conflict_policy: input.conflictPolicy
       });
     },

ui/koko/stores/fileTransfer.ts

@@ -7,7 +7,11 @@ import type {
 } from "@jumpserver/connectors-core";
 import { FileTransferUnavailableError, getFileTransferEndpoint } from "@jumpserver/connectors-core";
 import { defineStore } from "pinia";
-import { finalizeFileTransferChecksum, updateFileTransferChecksum } from "#koko/utils/file-transfer/checksum";
+import {
+  finalizeFileTransferChecksum,
+  isFileTransferChecksumState,
+  updateFileTransferChecksum
+} from "#koko/utils/file-transfer/checksum";
 import { loadFileTransferState, saveFileTransferState } from "#koko/utils/file-transfer/persistence";
 
 const resumableStatuses = new Set<FileTransferStatus>(["queued", "preparing", "transferring", "verifying"]);
@@ -360,7 +364,7 @@ export const useFileTransferStore = defineStore("file-transfer", () => {
 
       const checksumAligned =
         prepared.committedBytes === task.confirmedBytes &&
-        (prepared.committedBytes === 0 || Boolean(task.checksumState));
+        (prepared.committedBytes === 0 || isFileTransferChecksumState(task.checksumState));
       if (!checksumAligned) {
         await destination.cancelTransfer({
           transferId: task.id,

ui/koko/tests/sftp/fileTransferSha256.test.ts

@@ -1,5 +1,6 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
-import { sha256Hex } from "#koko/utils/file-transfer/sha256";
+import { isFileTransferChecksumState } from "#koko/utils/file-transfer/checksum";
+import { chainTransferChecksum, emptyTransferChecksumState, sha256Hex } from "#koko/utils/file-transfer/sha256";
 
 const abc = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
 const empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
@@ -19,4 +20,17 @@ describe("file transfer sha256", () => {
     expect(await sha256Hex(new TextEncoder().encode("abc"))).toBe(abc);
     expect(await sha256Hex(new Uint8Array())).toBe(empty);
   });
+
+  it("folds chunk hashes in order without re-hashing their contents", async () => {
+    let state = emptyTransferChecksumState();
+    state = await chainTransferChecksum(state, await sha256Hex(new TextEncoder().encode("hello")));
+    state = await chainTransferChecksum(state, await sha256Hex(new TextEncoder().encode("world")));
+    expect(state).toBe("98d128df384d428ffe76af3c0198ff1e8945ef71e741ba440bafff0510da8f22");
+  });
+
+  it("rejects corrupt persisted chain state", async () => {
+    expect(isFileTransferChecksumState(abc)).toBe(true);
+    expect(isFileTransferChecksumState('{"h":[]}')).toBe(false);
+    expect(() => chainTransferChecksum("broken", abc)).toThrow("Invalid file transfer checksum state");
+  });
 });

ui/koko/tests/sftp/fileTransferStore.test.ts

@@ -6,6 +6,7 @@ import { useFileTransferStore } from "#koko/stores/fileTransfer";
 import { loadFileTransferState } from "#koko/utils/file-transfer/persistence";
 
 vi.mock("#koko/utils/file-transfer/checksum", () => ({
+  isFileTransferChecksumState: vi.fn((state: string) => state === "checksum-state"),
   updateFileTransferChecksum: vi.fn(async () => ({ chunkChecksum: "chunk-checksum", state: "checksum-state" })),
   finalizeFileTransferChecksum: vi.fn(async () => "file-checksum")
 }));

ui/koko/tests/sftp/sftpTransportCore.test.ts

@@ -255,6 +255,40 @@ describe("useSftpTransferEndpoint transport core wiring", () => {
       data: JSON.stringify({ committed_bytes: 3, duplicate: false })
     });
     await expect(writePromise).resolves.toEqual({ committedBytes: 3, duplicate: false });
+
+    const commitPromise = endpoint.commitTransfer({
+      transferId: "transfer-a",
+      targetPath: "/remote/demo.txt",
+      totalBytes: 3,
+      sha256: "file-chain",
+      conflictPolicy: "overwrite"
+    });
+    expect(socket.send).toHaveBeenNthCalledWith(4, {
+      id: "request-4",
+      type: SftpMessageType.Data,
+      cmd: SftpCommand.TransferCommit,
+      data: JSON.stringify({
+        transfer_id: "transfer-a",
+        path: "/remote/demo.txt",
+        size: 3,
+        sha256: "file-chain",
+        chunk_size: 2 * 1024 * 1024,
+        conflict_policy: "overwrite"
+      }),
+      raw: ""
+    });
+    socket.emitMessage({
+      id: "request-4",
+      type: SftpMessageType.Data,
+      cmd: SftpCommand.TransferCommit,
+      data: JSON.stringify({
+        transfer_id: "transfer-a",
+        committed_bytes: 3,
+        total_bytes: 3,
+        state: "completed"
+      })
+    });
+    await expect(commitPromise).resolves.toBeUndefined();
   });
 
   it("sends transfer chunks as binary frames when the server advertises it", async () => {

ui/koko/utils/file-transfer/checksum.ts

@@ -45,6 +45,10 @@ function send(request: { kind: "update" | "finalize"; state: string; data?: Arra
   });
 }
 
+export function isFileTransferChecksumState(state: string) {
+  return /^[0-9a-f]{64}$/i.test(state);
+}
+
 export async function updateFileTransferChecksum(state: string, bytes: Uint8Array) {
   const copy = bytes.slice();
   const response = await send({ kind: "update", state, data: copy.buffer }, [copy.buffer]);

ui/koko/utils/file-transfer/fileTransferChecksum.worker.ts

@@ -1,4 +1,4 @@
-import { finalizeSha256, parseSha256State, sha256Hex, updateSha256 } from "./sha256";
+import { chainTransferChecksum, emptyTransferChecksumState, sha256Hex } from "./sha256";
 
 interface ChecksumRequest {
   id: string;
@@ -18,18 +18,21 @@ const workerScope = globalThis as unknown as ChecksumWorkerScope;
 workerScope.onmessage = async (event) => {
   try {
     const request = event.data;
-    const state = parseSha256State(request.state);
+    const state = request.state || emptyTransferChecksumState();
 
     if (request.kind === "finalize") {
-      workerScope.postMessage({ id: request.id, checksum: finalizeSha256(state), state: JSON.stringify(state) });
+      workerScope.postMessage({ id: request.id, checksum: state, state });
       return;
     }
 
     const bytes = new Uint8Array(request.data || new ArrayBuffer(0));
     const chunkChecksum = await sha256Hex(bytes);
 
-    updateSha256(state, bytes);
-    workerScope.postMessage({ id: request.id, chunkChecksum, state: JSON.stringify(state) });
+    workerScope.postMessage({
+      id: request.id,
+      chunkChecksum,
+      state: await chainTransferChecksum(state, chunkChecksum)
+    });
   } catch (error) {
     workerScope.postMessage({ id: event.data.id, error: error instanceof Error ? error.message : String(error) });
   }

ui/koko/utils/file-transfer/sha256.ts

@@ -134,3 +134,19 @@ export async function sha256Hex(bytes: Uint8Array) {
   }
   return sha256HexJs(bytes);
 }
+
+export function emptyTransferChecksumState() {
+  return "0".repeat(64);
+}
+
+export function chainTransferChecksum(state: string, chunkChecksum: string) {
+  if (!/^[0-9a-f]{64}$/i.test(state) || !/^[0-9a-f]{64}$/i.test(chunkChecksum)) {
+    throw new Error("Invalid file transfer checksum state");
+  }
+  const value = state + chunkChecksum;
+  const bytes = new Uint8Array(value.length / 2);
+  for (let index = 0; index < bytes.length; index++) {
+    bytes[index] = Number.parseInt(value.slice(index * 2, index * 2 + 2), 16);
+  }
+  return sha256Hex(bytes);
+}