Add CRC-64/NVME transactional checksum support for StageBlock

src/blob/handlers/BlockBlobHandler.ts

@@ -1,5 +1,7 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
 import {
+  computeTransactionalChecksums,
+  convertRawHeadersToMetadata,
+  getCRC64FromStream,
   getMD5FromStream,
   getMD5FromString,
   newEtag
@@ -187,6 +189,7 @@ export default class BlockBlobHandler
       ? options.transactionalContentMD5 ||
       context.request!.getHeader("content-md5")
       : undefined;
+    const contentCRC64 = options.transactionalContentCrc64;
 
     this.validateBlockId(blockId, blobCtx);
 
@@ -208,16 +211,31 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
-    const stream = await this.extentStore.readExtent(
-      persistency,
-      context.contextId
-    );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
+    // Only read the stored extent when at least one transactional checksum was provided.
+    // Compute only what is needed to avoid unnecessary CPU work.
+    let calculatedContentMD5: Uint8Array | undefined;
+    let calculatedCRC64: Uint8Array | undefined;
+
+    if (contentMD5 !== undefined || contentCRC64 !== undefined) {
+      const stream = await this.extentStore.readExtent(
+        persistency,
+        context.contextId
+      );
+      if (contentMD5 !== undefined && contentCRC64 !== undefined) {
+        const result = await computeTransactionalChecksums(stream);
+        calculatedContentMD5 = result.md5;
+        calculatedCRC64 = result.crc64;
+      } else if (contentMD5 !== undefined) {
+        calculatedContentMD5 = await getMD5FromStream(stream);
+      } else {
+        calculatedCRC64 = await getCRC64FromStream(stream);
+      }
+    }
+
     if (contentMD5 !== undefined) {
       if (typeof contentMD5 === "string") {
         const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
+          calculatedContentMD5!
         ).toString("base64");
         if (contentMD5 !== calculatedContentMD5String) {
           throw StorageErrorFactory.getInvalidOperation(
@@ -226,7 +244,7 @@ export default class BlockBlobHandler
           );
         }
       } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
+        if (!Buffer.from(contentMD5).equals(calculatedContentMD5!)) {
           throw StorageErrorFactory.getInvalidOperation(
             context.contextId!,
             "Provided contentMD5 doesn't match."
@@ -235,6 +253,15 @@ export default class BlockBlobHandler
       }
     }
 
+    if (contentCRC64 !== undefined) {
+      if (!Buffer.from(contentCRC64).equals(Buffer.from(calculatedCRC64!))) {
+        throw StorageErrorFactory.getInvalidOperation(
+          context.contextId!,
+          "Provided transactional CRC64 doesn't match."
+        );
+      }
+    }
+
     const block: BlockModel = {
       accountName,
       containerName,
@@ -255,6 +282,7 @@ export default class BlockBlobHandler
     const response: Models.BlockBlobStageBlockResponse = {
       statusCode: 201,
       contentMD5: undefined, // TODO: Block content MD5
+      xMsContentCrc64: contentCRC64 !== undefined ? calculatedCRC64 : undefined,
       requestId: blobCtx.contextId,
       version: BLOB_API_VERSION,
       date,

src/blob/middlewares/StrictModelMiddlewareFactory.ts

@@ -20,7 +20,6 @@ export const UnsupportedHeadersBlocker: StrictModelRequestValidator = async (
   logger: ILogger
 ): Promise<void> => {
   const UnsupportedHeaderKeys = [
-    HeaderConstants.X_MS_CONTENT_CRC64,
     HeaderConstants.X_MS_RANGE_GET_CONTENT_CRC64,
     HeaderConstants.X_MS_ENCRYPTION_KEY,
     HeaderConstants.X_MS_ENCRYPTION_KEY_SHA256,

src/common/utils/utils.ts

@@ -169,3 +169,97 @@ export async function getMD5FromStream(
       });
   });
 }
+
+// CRC-64/ECMA-182 implementation for Azure Storage transactional integrity checks.
+// Algorithm and lookup-table approach adapted from the Azure Storage JavaScript SDK (MIT License):
+// https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/storage-blob/src/utils/crc64.ts
+// Polynomial: 0x42F0E1EBA9EA3693 (ECMA-182 standard, unreflected, init=0, xorout=0)
+// Represented as two 32-bit halves (hi, lo) to avoid BigInt.
+const CRC64_POLY_HI = 0x42f0e1eb;
+const CRC64_POLY_LO = 0xa9ea3693;
+
+// Flat table: entry i occupies [i*2] (hi) and [i*2+1] (lo).
+const CRC64_TABLE: readonly number[] = (() => {
+  const table: number[] = new Array(512);
+  for (let i = 0; i < 256; i++) {
+    let hi = (i << 24) >>> 0;
+    let lo = 0;
+    for (let j = 0; j < 8; j++) {
+      if ((hi & 0x80000000) !== 0) {
+        hi = (((hi << 1) | (lo >>> 31)) ^ CRC64_POLY_HI) >>> 0;
+        lo = ((lo << 1) ^ CRC64_POLY_LO) >>> 0;
+      } else {
+        hi = ((hi << 1) | (lo >>> 31)) >>> 0;
+        lo = (lo << 1) >>> 0;
+      }
+    }
+    table[i * 2] = hi;
+    table[i * 2 + 1] = lo;
+  }
+  return table;
+})();
+
+function crc64Accumulate(
+  crcHi: number, crcLo: number, chunk: Uint8Array
+): [number, number] {
+  for (let i = 0; i < chunk.length; i++) {
+    const index = ((crcHi >>> 24) ^ chunk[i]) & 0xff;
+    const tHi = CRC64_TABLE[index * 2];
+    const tLo = CRC64_TABLE[index * 2 + 1];
+    crcHi = (((crcHi << 8) | (crcLo >>> 24)) ^ tHi) >>> 0;
+    crcLo = ((crcLo << 8) ^ tLo) >>> 0;
+  }
+  return [crcHi, crcLo];
+}
+
+function crc64ToUint8Array(hi: number, lo: number): Uint8Array {
+  const buf = Buffer.allocUnsafe(8);
+  buf.writeUInt32BE(hi >>> 0, 0);
+  buf.writeUInt32BE(lo >>> 0, 4);
+  return buf;
+}
+
+export function getCRC64FromString(text: string): Uint8Array {
+  const [hi, lo] = crc64Accumulate(0, 0, Buffer.from(text));
+  return crc64ToUint8Array(hi, lo);
+}
+
+export async function getCRC64FromStream(
+  stream: NodeJS.ReadableStream
+): Promise<Uint8Array> {
+  return new Promise<Uint8Array>((resolve, reject) => {
+    let hi = 0, lo = 0;
+    stream
+      .on("data", (chunk: Buffer | string) => {
+        const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
+        [hi, lo] = crc64Accumulate(hi, lo, data);
+      })
+      .on("end", () => {
+        resolve(crc64ToUint8Array(hi, lo));
+      })
+      .on("error", reject);
+  });
+}
+
+/**
+ * Computes MD5 and CRC-64/ECMA-182 in a single stream pass, avoiding
+ * reading the extent twice when both checksums may be needed.
+ */
+export async function computeTransactionalChecksums(
+  stream: NodeJS.ReadableStream
+): Promise<{ md5: Uint8Array; crc64: Uint8Array }> {
+  const hash = createHash("md5");
+  return new Promise((resolve, reject) => {
+    let hi = 0, lo = 0;
+    stream
+      .on("data", (chunk: Buffer | string) => {
+        const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
+        hash.update(data);
+        [hi, lo] = crc64Accumulate(hi, lo, data);
+      })
+      .on("end", () => {
+        resolve({ md5: hash.digest(), crc64: crc64ToUint8Array(hi, lo) });
+      })
+      .on("error", reject);
+  });
+}

tests/blob/apis/blockblob.test.ts

@@ -18,7 +18,10 @@ import {
   getUniqueName,
   sleep
 } from "../../testutils";
-import { getMD5FromString } from "../../../src/common/utils/utils";
+import {
+  getCRC64FromString,
+  getMD5FromString
+} from "../../../src/common/utils/utils";
 
 // Set true to enable debug log
 configLogger(false);
@@ -316,6 +319,74 @@ describe("BlockBlobAPIs", () => {
     assert.equal(listResponse.uncommittedBlocks![0].size, body.length);
   });
 
+  it("stageBlock with correct crc64 should succeed @loki @sql", async () => {
+    const body = "HelloWorld";
+    const crc64 = getCRC64FromString(body);
+    const options = { transactionalContentCrc64: new Uint8Array(crc64) };
+
+    const result = await blockBlobClient.stageBlock(
+      base64encode("1"),
+      body,
+      body.length,
+      options
+    );
+
+    assert.equal(result._response.status, 201);
+    // Server must echo back the CRC64 it validated against
+    assert.ok(
+      result.xMsContentCrc64 !== undefined,
+      "Response should include x-ms-content-crc64"
+    );
+    assert.deepStrictEqual(
+      Buffer.from(result.xMsContentCrc64!),
+      Buffer.from(crc64),
+      "Echoed CRC64 must match what was sent"
+    );
+
+    const listResponse = await blockBlobClient.getBlockList("uncommitted");
+    assert.equal(listResponse.uncommittedBlocks!.length, 1);
+    assert.equal(listResponse.uncommittedBlocks![0].name, base64encode("1"));
+    assert.equal(listResponse.uncommittedBlocks![0].size, body.length);
+  });
+
+  it("stageBlock with wrong body should throw crc64 mismatch @loki @sql", async () => {
+    const body = "HelloWorld";
+    // Provide CRC64 of a different payload — server must reject the upload
+    const wrongCrc64 = getCRC64FromString("differentBody");
+    const options = { transactionalContentCrc64: new Uint8Array(wrongCrc64) };
+
+    try {
+      await blockBlobClient.stageBlock(
+        base64encode("1"),
+        body,
+        body.length,
+        options
+      );
+    } catch (e) {
+      assert.equal(e.name, "RestError");
+      assert.equal(e.statusCode, 400);
+      assert.equal(
+        e.details.message.indexOf("Provided transactional CRC64 doesn't match."),
+        0
+      );
+      return;
+    }
+    assert.fail("Did not throw an exception.");
+  });
+
+  it("stageBlock without crc64 header should not include crc64 in response @loki @sql", async () => {
+    // When no x-ms-content-crc64 is sent the response must not include one,
+    // matching the behaviour of the real service.
+    const body = "HelloWorld";
+    const result = await blockBlobClient.stageBlock(
+      base64encode("1"),
+      body,
+      body.length
+    );
+    assert.equal(result._response.status, 201);
+    assert.strictEqual(result.xMsContentCrc64, undefined);
+  });
+
   it("commitBlockList @loki @sql", async () => {
     const body = "HelloWorld";
     await blockBlobClient.stageBlock(base64encode("1"), body, body.length);

tests/blob/utils.test.ts

@@ -1,5 +1,10 @@
 import * as assert from "assert";
-import { convertRawHeadersToMetadata } from "../../src/common/utils/utils";
+import { PassThrough } from "stream";
+import {
+  convertRawHeadersToMetadata,
+  getCRC64FromStream,
+  getCRC64FromString
+} from "../../src/common/utils/utils";
 
 describe("Utils", () => {
   it("convertRawHeadersToMetadata should work", () => {
@@ -57,3 +62,58 @@ describe("Utils", () => {
     assert.deepStrictEqual(metadata, undefined);
   });
 });
+
+describe("CRC64", () => {
+  // CRC-64/ECMA-182 check value for "123456789" per the CRC catalogue:
+  // https://reveng.sourceforge.io/crc-catalogue/all.htm
+  it("getCRC64FromString matches the standard CRC-64/ECMA-182 check value for '123456789'", () => {
+    const result = getCRC64FromString("123456789");
+    const hex = Buffer.from(result).toString("hex");
+    assert.strictEqual(hex, "6c40df5f0b497347");
+  });
+
+  it("getCRC64FromString produces an 8-byte result", () => {
+    assert.strictEqual(getCRC64FromString("").length, 8);
+    assert.strictEqual(getCRC64FromString("Hello, World!").length, 8);
+  });
+
+  it("getCRC64FromStream matches getCRC64FromString for the same data", async () => {
+    const data = "The quick brown fox jumps over the lazy dog";
+    const fromString = getCRC64FromString(data);
+
+    const stream = new PassThrough();
+    stream.end(Buffer.from(data));
+    const fromStream = await getCRC64FromStream(stream);
+
+    assert.deepStrictEqual(Buffer.from(fromString), Buffer.from(fromStream));
+  });
+
+  it("getCRC64FromStream produces identical results regardless of chunk boundaries", async () => {
+    // Streaming data split across different chunk sizes must produce the same
+    // CRC as a single contiguous buffer — chunk boundaries must not affect the result.
+    const data = Buffer.from("Azure Blob Storage block integrity check");
+    const expected = getCRC64FromString(data.toString());
+
+    // Push as many 3-byte chunks (deliberately misaligned with any word boundary)
+    const chunked = new PassThrough();
+    for (let i = 0; i < data.length; i += 3) {
+      chunked.push(data.slice(i, i + 3));
+    }
+    chunked.push(null);
+    const fromChunked = await getCRC64FromStream(chunked);
+
+    assert.deepStrictEqual(Buffer.from(fromChunked), Buffer.from(expected));
+  });
+
+  it("getCRC64FromString produces distinct values for inputs that differ by a single byte", () => {
+    // Verifies the avalanche property: a one-byte change must alter the checksum.
+    const base = Buffer.from("block content for crc64 test");
+    const mutated = Buffer.from(base);
+    mutated[mutated.length - 1] ^= 0x01;
+
+    const crc1 = getCRC64FromString(base.toString("latin1"));
+    const crc2 = getCRC64FromString(mutated.toString("latin1"));
+
+    assert.notDeepStrictEqual(Buffer.from(crc1), Buffer.from(crc2));
+  });
+});

tests/testutils.ts

@@ -10,9 +10,14 @@ export const EMULATOR_ACCOUNT_NAME = "devstoreaccount1";
 export const EMULATOR_ACCOUNT_KEY =
   "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
 
+// Counter-based suffix instead of Math.random() to guarantee uniqueness within
+// a test run. Random suffixes can collide when multiple entities are created
+// within the same millisecond on fast CI runners, causing flaky batch tests.
+let _uniqueNameCounter = 0;
+
 export function getUniqueName(prefix: string): string {
   return `${prefix}${new Date().getTime()}${padStart(
-    Math.floor(Math.random() * 10000).toString(),
+    (++_uniqueNameCounter).toString(),
     5,
     "00000"
   )}`;