Add CRC-64/NVME transactional checksum support for StageBlock

src/blob/errors/StorageErrorFactory.ts

@@ -190,6 +190,43 @@ export default class StorageErrorFactory {
     );
   }
 
+  public static getCrc64Mismatch(
+    contextID: string = DefaultID,
+    userSpecifiedCrc64: string,
+    serverCalculatedCrc64: string
+  ): StorageError {
+    return new StorageError(
+      400,
+      "Crc64Mismatch",
+      "The CRC64 value specified in the request did not match with the CRC64 value calculated by the server.",
+      contextID,
+      {
+        UserSpecifiedCrc64: userSpecifiedCrc64,
+        ServerCalculatedCrc64: serverCalculatedCrc64
+      }
+    );
+  }
+
+  public static getBothCrc64AndMd5HeaderPresent(
+    contextID: string = DefaultID
+  ): StorageError {
+    return new StorageError(
+      400,
+      "BothCrc64AndMd5HeaderPresent",
+      "Both x-ms-content-crc64 header and Content-MD5 header are present.",
+      contextID
+    );
+  }
+
+  public static getInvalidMd5(contextID: string = DefaultID): StorageError {
+    return new StorageError(
+      400,
+      "InvalidMd5",
+      "The MD5 value specified in the request is invalid. The MD5 value must be 128 bits and Base64-encoded.",
+      contextID
+    );
+  }
+
   public static getInvalidPageRange(contextID: string): StorageError {
     return new StorageError(
       416,

src/blob/handlers/AppendBlobHandler.ts

@@ -1,5 +1,7 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
-import { getMD5FromStream, newEtag } from "../../common/utils/utils";
+import {
+  convertRawHeadersToMetadata,
+  newEtag
+} from "../../common/utils/utils";
 import BlobStorageContext from "../context/BlobStorageContext";
 import NotImplementedError from "../errors/NotImplementedError";
 import StorageErrorFactory from "../errors/StorageErrorFactory";
@@ -13,7 +15,7 @@ import {
   MAX_APPEND_BLOB_BLOCK_COUNT,
   MAX_APPEND_BLOB_BLOCK_SIZE
 } from "../utils/constants";
-import { getTagsFromString } from "../utils/utils";
+import { computeAndValidateTransactionalChecksums, getTagsFromString } from "../utils/utils";
 import BaseHandler from "./BaseHandler";
 
 export default class AppendBlobHandler extends BaseHandler
@@ -149,38 +151,28 @@ export default class AppendBlobHandler extends BaseHandler
       );
     }
 
-    // MD5
+    // MD5 and/or CRC64 transactional integrity validation
     const contentMD5 = blobCtx.request!.getHeader(HeaderConstants.CONTENT_MD5);
+    const contentCRC64 = options.transactionalContentCrc64;
     let contentMD5Buffer;
-    let contentMD5String;
 
     if (contentMD5 !== undefined) {
       contentMD5Buffer =
         typeof contentMD5 === "string"
           ? Buffer.from(contentMD5, "base64")
           : contentMD5;
-      contentMD5String =
-        typeof contentMD5 === "string"
-          ? contentMD5
-          : contentMD5Buffer.toString("base64");
+    }
 
-      const stream = await this.extentStore.readExtent(
-        extent,
-        blobCtx.contextId
+    // Per the Append Block REST contract, the service always computes a CRC64
+    // of the appended block and returns it in x-ms-content-crc64.
+    const stream = await this.extentStore.readExtent(extent, blobCtx.contextId);
+    const { crc64: calculatedCRC64 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { crc64: true }
       );
-      const calculatedContentMD5Buffer = await getMD5FromStream(stream);
-      const calculatedContentMD5String = Buffer.from(
-        calculatedContentMD5Buffer
-      ).toString("base64");
-
-      if (contentMD5String !== calculatedContentMD5String) {
-        throw StorageErrorFactory.getMd5Mismatch(
-          context.contextId,
-          contentMD5String,
-          calculatedContentMD5String
-        );
-      }
-    }
 
     const originOffset = blob.properties.contentLength;
 
@@ -206,7 +198,7 @@ export default class AppendBlobHandler extends BaseHandler
       eTag: properties.etag,
       lastModified: properties.lastModified,
       contentMD5: contentMD5Buffer,
-      xMsContentCrc64: undefined,
+      xMsContentCrc64: calculatedCRC64,
       clientRequestId: options.requestId,
       version: BLOB_API_VERSION,
       date,

src/blob/handlers/BlobHandler.ts

@@ -907,6 +907,9 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler {
       date: context.startTime,
       copyId: res.copyId,
       copyStatus,
+      // Per the Copy Blob From URL REST contract, echo the source's Content-MD5
+      // back to the client when it was supplied in x-ms-source-content-md5.
+      contentMD5: options.sourceContentMD5,
       clientRequestId: options.requestId
     };
 

src/blob/handlers/BlockBlobHandler.ts

@@ -1,6 +1,5 @@
-import { convertRawHeadersToMetadata } from "../../common/utils/utils";
 import {
-  getMD5FromStream,
+  convertRawHeadersToMetadata,
   getMD5FromString,
   newEtag
 } from "../../common/utils/utils";
@@ -14,7 +13,7 @@ import { parseXML } from "../generated/utils/xml";
 import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore";
 import { BLOB_API_VERSION } from "../utils/constants";
 import BaseHandler from "./BaseHandler";
-import { getTagsFromString } from "../utils/utils";
+import { computeAndValidateTransactionalChecksums, getTagsFromString } from "../utils/utils";
 
 /**
  * BlobHandler handles Azure Storage BlockBlob related requests.
@@ -50,6 +49,7 @@ export default class BlockBlobHandler
       ? options.blobHTTPHeaders.blobContentMD5 ||
       context.request!.getHeader("content-md5")
       : undefined;
+    const contentCRC64 = options.transactionalContentCrc64;
 
     await this.metadataStore.checkContainerExist(
       context,
@@ -68,32 +68,19 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
+    // MD5 is always needed (persisted as the blob's contentMD5 property);
+    // CRC64 is computed in the same pass only when the client supplied one.
     const stream = await this.extentStore.readExtent(
       persistency,
       context.contextId
     );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
-    if (contentMD5 !== undefined) {
-      if (typeof contentMD5 === "string") {
-        const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
-        ).toString("base64");
-        if (contentMD5 !== calculatedContentMD5String) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      }
-    }
+    const { md5: calculatedContentMD5 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { md5: true }
+      );
 
     const blob: BlobModel = {
       deleted: false,
@@ -187,6 +174,7 @@ export default class BlockBlobHandler
       ? options.transactionalContentMD5 ||
       context.request!.getHeader("content-md5")
       : undefined;
+    const contentCRC64 = options.transactionalContentCrc64;
 
     this.validateBlockId(blockId, blobCtx);
 
@@ -208,32 +196,22 @@ export default class BlockBlobHandler
       );
     }
 
-    // Calculate MD5 for validation
+    // Per the Put Block REST contract, the service computes a CRC64 of the
+    // staged block and echoes it back in x-ms-content-crc64 unless the client
+    // supplied a Content-MD5 (Azure rejects supplying both). Compute CRC64
+    // whenever no MD5 was supplied, regardless of whether the client supplied
+    // a CRC64 themselves.
     const stream = await this.extentStore.readExtent(
       persistency,
       context.contextId
     );
-    const calculatedContentMD5 = await getMD5FromStream(stream);
-    if (contentMD5 !== undefined) {
-      if (typeof contentMD5 === "string") {
-        const calculatedContentMD5String = Buffer.from(
-          calculatedContentMD5
-        ).toString("base64");
-        if (contentMD5 !== calculatedContentMD5String) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      } else {
-        if (!Buffer.from(contentMD5).equals(calculatedContentMD5)) {
-          throw StorageErrorFactory.getInvalidOperation(
-            context.contextId!,
-            "Provided contentMD5 doesn't match."
-          );
-        }
-      }
-    }
+    const { crc64: calculatedCRC64 } =
+      await computeAndValidateTransactionalChecksums(
+        stream,
+        { md5: contentMD5, crc64: contentCRC64 },
+        context.contextId,
+        { crc64: contentMD5 === undefined }
+      );
 
     const block: BlockModel = {
       accountName,
@@ -255,6 +233,7 @@ export default class BlockBlobHandler
     const response: Models.BlockBlobStageBlockResponse = {
       statusCode: 201,
       contentMD5: undefined, // TODO: Block content MD5
+      xMsContentCrc64: calculatedCRC64,
       requestId: blobCtx.contextId,
       version: BLOB_API_VERSION,
       date,

src/blob/middlewares/StrictModelMiddlewareFactory.ts

@@ -20,7 +20,6 @@ export const UnsupportedHeadersBlocker: StrictModelRequestValidator = async (
   logger: ILogger
 ): Promise<void> => {
   const UnsupportedHeaderKeys = [
-    HeaderConstants.X_MS_CONTENT_CRC64,
     HeaderConstants.X_MS_RANGE_GET_CONTENT_CRC64,
     HeaderConstants.X_MS_ENCRYPTION_KEY,
     HeaderConstants.X_MS_ENCRYPTION_KEY_SHA256,

src/blob/utils/utils.ts

@@ -4,6 +4,61 @@ import StorageErrorFactory from "../errors/StorageErrorFactory";
 import { USERDELEGATIONKEY_BASIC_KEY } from "./constants";
 import { BlobTag, BlobTags } from "@azure/storage-blob";
 import { TagContent } from "../persistence/QueryInterpreter/QueryNodes/IQueryNode";
+import { computeTransactionalChecksums } from "../../common/utils/utils";
+
+/**
+ * Computes MD5 and/or CRC-64/NVME from a stream in a single pass and validates
+ * against the request-supplied values. Throws Md5Mismatch / Crc64Mismatch
+ * (HTTP 400) on mismatch — the documented Azure Storage error codes for
+ * transactional integrity failures.
+ *
+ * Rejects requests that supply both checksums with `BothCrc64AndMd5HeaderPresent`
+ * (HTTP 400), matching the real Azure service contract.
+ *
+ * A checksum is computed when its `expected` value is provided, OR when the
+ * corresponding `force` flag is set (for callers that need the value for
+ * non-validation purposes — e.g. Put Blob persists MD5 as a blob property).
+ */
+export async function computeAndValidateTransactionalChecksums(
+  stream: NodeJS.ReadableStream,
+  expected: { md5?: Uint8Array | string; crc64?: Uint8Array },
+  contextId: string | undefined,
+  force?: { md5?: boolean; crc64?: boolean }
+): Promise<{ md5?: Uint8Array; crc64?: Uint8Array }> {
+  if (expected.md5 !== undefined && expected.crc64 !== undefined) {
+    throw StorageErrorFactory.getBothCrc64AndMd5HeaderPresent(contextId);
+  }
+  if (expected.md5 !== undefined) {
+    const md5Bytes =
+      typeof expected.md5 === "string"
+        ? Buffer.from(expected.md5, "base64")
+        : Buffer.from(expected.md5);
+    if (md5Bytes.length !== 16) {
+      throw StorageErrorFactory.getInvalidMd5(contextId);
+    }
+  }
+  const calculated = await computeTransactionalChecksums(stream, expected, force);
+
+  if (expected.md5 !== undefined) {
+    const expectedMd5 =
+      typeof expected.md5 === "string"
+        ? expected.md5
+        : Buffer.from(expected.md5).toString("base64");
+    const calculatedMd5 = Buffer.from(calculated.md5!).toString("base64");
+    if (expectedMd5 !== calculatedMd5) {
+      throw StorageErrorFactory.getMd5Mismatch(contextId, expectedMd5, calculatedMd5);
+    }
+  }
+  if (expected.crc64 !== undefined) {
+    const expectedCrc64 = Buffer.from(expected.crc64).toString("base64");
+    const calculatedCrc64 = Buffer.from(calculated.crc64!).toString("base64");
+    if (expectedCrc64 !== calculatedCrc64) {
+      throw StorageErrorFactory.getCrc64Mismatch(contextId, expectedCrc64, calculatedCrc64);
+    }
+  }
+
+  return calculated;
+}
 
 export function checkApiVersion(
   inputApiVersion: string,