Add CRC-64/NVME transactional checksum support for StageBlock

src/blob/handlers/BlockBlobHandler.ts

@@ -1,5 +1,6 @@
 import { convertRawHeadersToMetadata } from "../../common/utils/utils";
 import {
+  computeTransactionalChecksums,
   getMD5FromStream,
   getMD5FromString,
   newEtag
@@ -187,6 +188,7 @@ export default class BlockBlobHandler
       ? options.transactionalContentMD5 ||
       context.request!.getHeader("content-md5")
       : undefined;
+    const contentCRC64 = options.transactionalContentCrc64;
 
     this.validateBlockId(blockId, blobCtx);
 
@@ -208,12 +210,14 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
+    // Compute MD5 and CRC64 in a single pass over the stored extent
     const stream = await this.extentStore.readExtent(
       persistency,
       context.contextId
     );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
+    const { md5: calculatedContentMD5, crc64: calculatedCRC64 } =
+      await computeTransactionalChecksums(stream);
+
     if (contentMD5 !== undefined) {
       if (typeof contentMD5 === "string") {
         const calculatedContentMD5String = Buffer.from(
@@ -235,6 +239,15 @@ export default class BlockBlobHandler
       }
     }
 
+    if (contentCRC64 !== undefined) {
+      if (!Buffer.from(contentCRC64).equals(Buffer.from(calculatedCRC64))) {
+        throw StorageErrorFactory.getInvalidOperation(
+          context.contextId!,
+          "Provided transactional CRC64 doesn't match."
+        );
+      }
+    }
+
     const block: BlockModel = {
       accountName,
       containerName,
@@ -255,6 +268,7 @@ export default class BlockBlobHandler
     const response: Models.BlockBlobStageBlockResponse = {
       statusCode: 201,
       contentMD5: undefined, // TODO: Block content MD5
+      xMsContentCrc64: contentCRC64 !== undefined ? calculatedCRC64 : undefined,
       requestId: blobCtx.contextId,
       version: BLOB_API_VERSION,
       date,

src/blob/middlewares/StrictModelMiddlewareFactory.ts

@@ -20,7 +20,6 @@ export const UnsupportedHeadersBlocker: StrictModelRequestValidator = async (
   logger: ILogger
 ): Promise<void> => {
   const UnsupportedHeaderKeys = [
-    HeaderConstants.X_MS_CONTENT_CRC64,
     HeaderConstants.X_MS_RANGE_GET_CONTENT_CRC64,
     HeaderConstants.X_MS_ENCRYPTION_KEY,
     HeaderConstants.X_MS_ENCRYPTION_KEY_SHA256,

src/common/utils/utils.ts

@@ -169,3 +169,84 @@ export async function getMD5FromStream(
       });
   });
 }
+
+// CRC-64/ECMA-182 implementation for Azure Storage transactional integrity checks.
+// Algorithm and lookup-table approach adapted from the Azure Storage JavaScript SDK (MIT License):
+// https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/storage-blob/src/utils/crc64.ts
+// Polynomial: 0x42F0E1EBA9EA3693 (ECMA-182 standard, unreflected, init=0, xorout=0)
+const CRC64_POLY = 0x42f0e1eba9ea3693n;
+
+const CRC64_TABLE: readonly bigint[] = (() => {
+  const table: bigint[] = new Array(256);
+  for (let i = 0; i < 256; i++) {
+    let crc = BigInt(i) << 56n;
+    for (let j = 0; j < 8; j++) {
+      if ((crc & 0x8000000000000000n) !== 0n) {
+        crc = ((crc << 1n) ^ CRC64_POLY) & 0xffffffffffffffffn;
+      } else {
+        crc = (crc << 1n) & 0xffffffffffffffffn;
+      }
+    }
+    table[i] = crc;
+  }
+  return table;
+})();
+
+function crc64Accumulate(crc: bigint, chunk: Uint8Array): bigint {
+  for (let i = 0; i < chunk.length; i++) {
+    const index = Number((crc >> 56n) ^ BigInt(chunk[i])) & 0xff;
+    crc = ((crc << 8n) ^ CRC64_TABLE[index]) & 0xffffffffffffffffn;
+  }
+  return crc;
+}
+
+function bigintToUint8Array(n: bigint): Uint8Array {
+  const buf = Buffer.allocUnsafe(8);
+  buf.writeUInt32BE(Number(n >> 32n) >>> 0, 0);
+  buf.writeUInt32BE(Number(n & 0xffffffffn) >>> 0, 4);
+  return buf;
+}
+
+export function getCRC64FromString(text: string): Uint8Array {
+  return bigintToUint8Array(crc64Accumulate(0n, Buffer.from(text)));
+}
+
+export async function getCRC64FromStream(
+  stream: NodeJS.ReadableStream
+): Promise<Uint8Array> {
+  return new Promise<Uint8Array>((resolve, reject) => {
+    let crc = 0n;
+    stream
+      .on("data", (chunk: Buffer | string) => {
+        const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
+        crc = crc64Accumulate(crc, data);
+      })
+      .on("end", () => {
+        resolve(bigintToUint8Array(crc));
+      })
+      .on("error", reject);
+  });
+}
+
+/**
+ * Computes MD5 and CRC-64/ECMA-182 in a single stream pass, avoiding
+ * reading the extent twice when both checksums may be needed.
+ */
+export async function computeTransactionalChecksums(
+  stream: NodeJS.ReadableStream
+): Promise<{ md5: Uint8Array; crc64: Uint8Array }> {
+  const hash = createHash("md5");
+  return new Promise((resolve, reject) => {
+    let crc = 0n;
+    stream
+      .on("data", (chunk: Buffer | string) => {
+        const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
+        hash.update(data);
+        crc = crc64Accumulate(crc, data);
+      })
+      .on("end", () => {
+        resolve({ md5: hash.digest(), crc64: bigintToUint8Array(crc) });
+      })
+      .on("error", reject);
+  });
+}

tests/blob/apis/blockblob.test.ts

@@ -18,7 +18,10 @@ import {
   getUniqueName,
   sleep
 } from "../../testutils";
-import { getMD5FromString } from "../../../src/common/utils/utils";
+import {
+  getCRC64FromString,
+  getMD5FromString
+} from "../../../src/common/utils/utils";
 
 // Set true to enable debug log
 configLogger(false);
@@ -316,6 +319,74 @@ describe("BlockBlobAPIs", () => {
     assert.equal(listResponse.uncommittedBlocks![0].size, body.length);
   });
 
+  it("stageBlock with correct crc64 should succeed @loki @sql", async () => {
+    const body = "HelloWorld";
+    const crc64 = getCRC64FromString(body);
+    const options = { transactionalContentCrc64: new Uint8Array(crc64) };
+
+    const result = await blockBlobClient.stageBlock(
+      base64encode("1"),
+      body,
+      body.length,
+      options
+    );
+
+    assert.equal(result._response.status, 201);
+    // Server must echo back the CRC64 it validated against
+    assert.ok(
+      result.xMsContentCrc64 !== undefined,
+      "Response should include x-ms-content-crc64"
+    );
+    assert.deepStrictEqual(
+      Buffer.from(result.xMsContentCrc64!),
+      Buffer.from(crc64),
+      "Echoed CRC64 must match what was sent"
+    );
+
+    const listResponse = await blockBlobClient.getBlockList("uncommitted");
+    assert.equal(listResponse.uncommittedBlocks!.length, 1);
+    assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("1"));
+    assert.equal(listResponse.uncommittedBlocks![0].size, body.length);
+  });
+
+  it("stageBlock with wrong body should throw crc64 mismatch @loki @sql", async () => {
+    const body = "HelloWorld";
+    // Provide CRC64 of a different payload — server must reject the upload
+    const wrongCrc64 = getCRC64FromString("differentBody");
+    const options = { transactionalContentCrc64: new Uint8Array(wrongCrc64) };
+
+    try {
+      await blockBlobClient.stageBlock(
+        base64encode("1"),
+        body,
+        body.length,
+        options
+      );
+    } catch (e) {
+      assert.equal(e.name, "RestError");
+      assert.equal(e.statusCode, 400);
+      assert.equal(
+        e.details.message.indexOf("Provided transactional CRC64 doesn't match."),
+        0
+      );
+      return;
+    }
+    assert.fail("Did not throw an exception.");
+  });
+
+  it("stageBlock without crc64 header should not include crc64 in response @loki @sql", async () => {
+    // When no x-ms-content-crc64 is sent the response must not include one,
+    // matching the behaviour of the real service.
+    const body = "HelloWorld";
+    const result = await blockBlobClient.stageBlock(
+      base64encode("1"),
+      body,
+      body.length
+    );
+    assert.equal(result._response.status, 201);
+    assert.strictEqual(result.xMsContentCrc64, undefined);
+  });
+
   it("commitBlockList @loki @sql", async () => {
     const body = "HelloWorld";
     await blockBlobClient.stageBlock(base64encode("1"), body, body.length);

tests/blob/utils.test.ts

@@ -1,5 +1,10 @@
 import * as assert from "assert";
-import { convertRawHeadersToMetadata } from "../../src/common/utils/utils";
+import { PassThrough } from "stream";
+import {
+  convertRawHeadersToMetadata,
+  getCRC64FromStream,
+  getCRC64FromString
+} from "../../src/common/utils/utils";
 
 describe("Utils", () => {
   it("convertRawHeadersToMetadata should work", () => {
@@ -57,3 +62,58 @@ describe("Utils", () => {
     assert.deepStrictEqual(metadata, undefined);
   });
 });
+
+describe("CRC64", () => {
+  // CRC-64/ECMA-182 check value for "123456789" per the CRC catalogue:
+  // https://reveng.sourceforge.io/crc-catalogue/all.htm
+  it("getCRC64FromString matches the standard CRC-64/ECMA-182 check value for '123456789'", () => {
+    const result = getCRC64FromString("123456789");
+    const hex = Buffer.from(result).toString("hex");
+    assert.strictEqual(hex, "6c40df5f0b497347");
+  });
+
+  it("getCRC64FromString produces an 8-byte result", () => {
+    assert.strictEqual(getCRC64FromString("").length, 8);
+    assert.strictEqual(getCRC64FromString("Hello, World!").length, 8);
+  });
+
+  it("getCRC64FromStream matches getCRC64FromString for the same data", async () => {
+    const data = "The quick brown fox jumps over the lazy dog";
+    const fromString = getCRC64FromString(data);
+
+    const stream = new PassThrough();
+    stream.end(Buffer.from(data));
+    const fromStream = await getCRC64FromStream(stream);
+
+    assert.deepStrictEqual(Buffer.from(fromString), Buffer.from(fromStream));
+  });
+
+  it("getCRC64FromStream produces identical results regardless of chunk boundaries", async () => {
+    // Streaming data split across different chunk sizes must produce the same
+    // CRC as a single contiguous buffer — chunk boundaries must not affect the result.
+    const data = Buffer.from("Azure Blob Storage block integrity check");
+    const expected = getCRC64FromString(data.toString());
+
+    // Push as many 3-byte chunks (deliberately misaligned with any word boundary)
+    const chunked = new PassThrough();
+    for (let i = 0; i < data.length; i += 3) {
+      chunked.push(data.slice(i, i + 3));
+    }
+    chunked.push(null);
+    const fromChunked = await getCRC64FromStream(chunked);
+
+    assert.deepStrictEqual(Buffer.from(fromChunked), Buffer.from(expected));
+  });
+
+  it("getCRC64FromString produces distinct values for inputs that differ by a single byte", () => {
+    // Verifies the avalanche property: a one-byte change must alter the checksum.
+    const base = Buffer.from("block content for crc64 test");
+    const mutated = Buffer.from(base);
+    mutated[mutated.length - 1] ^= 0x01;
+
+    const crc1 = getCRC64FromString(base.toString("latin1"));
+    const crc2 = getCRC64FromString(mutated.toString("latin1"));
+
+    assert.notDeepStrictEqual(Buffer.from(crc1), Buffer.from(crc2));
+  });
+});