Remove legacy SHA256 support, add CRC64NVME type

- Remove plain SHA256 checksum support (always use SHA256_CHUNKED)
- Add CHECKSUM_TYPE_CRC64NVME to model for reading packages
- Remove conditional file size logic (always 5TB now that chunked is default)
- Rename config flag: chunkedChecksums -> crc64Checksums (for future use)
- Delete checksums-legacy.spec.ts (plain SHA256 tests)
- Clean up test mocks and suite names

The catalog always computes SHA256_CHUNKED checksums for uploads.
The crc64Checksums flag is reserved for future catalog-side CRC64NVME
computation, but currently doesn't affect behavior.

Author: nl0

catalog/app/containers/Bucket/PackageDialog/Inputs/Files/constants.ts

@@ -1,9 +1,5 @@
-import cfg from 'constants/config'
-
 export const MAX_UPLOAD_SIZE = 20 * 1000 * 1000 * 1000 // 20GB
 // XXX: keep in sync w/ the backend
 // NOTE: these limits are lower than the actual "hard" limits on the backend
-export const MAX_S3_SIZE = cfg.chunkedChecksums
-  ? 5 * 10 ** 12 // 5 TB
-  : 50 * 10 ** 9 // 50 GB
+export const MAX_S3_SIZE = 5 * 10 ** 12 // 5 TB
 export const MAX_FILE_COUNT = 1000

catalog/app/model/index.ts

@@ -50,8 +50,17 @@ export type EntryMeta = (Types.JsonRecord & { user_meta?: Types.JsonRecord }) |
 
 export const CHECKSUM_TYPE_SHA256 = 'SHA256' as const
 export const CHECKSUM_TYPE_SHA256_CHUNKED = 'sha2-256-chunked' as const
+export const CHECKSUM_TYPE_CRC64NVME = 'CRC64NVME' as const
+export const CHECKSUM_TYPES = [
+  CHECKSUM_TYPE_SHA256,
+  CHECKSUM_TYPE_SHA256_CHUNKED,
+  CHECKSUM_TYPE_CRC64NVME,
+]
+
+export type ChecksumType = (typeof CHECKSUM_TYPES)[number]
+
 export interface Checksum {
-  type: typeof CHECKSUM_TYPE_SHA256 | typeof CHECKSUM_TYPE_SHA256_CHUNKED
+  type: ChecksumType
   value: string
 }
 

catalog/app/utils/Config.ts

@@ -43,7 +43,8 @@ export interface ConfigJson {
   ssoAuth: AuthMethodConfig
   ssoProviders: string
 
-  chunkedChecksums?: boolean
+  // NOTE: does not affect anything at the moment
+  crc64Checksums?: boolean
 
   qurator?: boolean
 
@@ -94,7 +95,7 @@ const transformConfig = (cfg: ConfigJson) => ({
   noOverviewImages: !!cfg.noOverviewImages,
   /** @deprecated */
   desktop: !!cfg.desktop,
-  chunkedChecksums: !!cfg.chunkedChecksums,
+  crc64Checksums: !!cfg.crc64Checksums,
   qurator: !!cfg.qurator,
 })
 

catalog/app/utils/checksums/checksums-legacy.spec.ts

@@ -4,15 +4,10 @@ import util from 'util'
 
 import computeFileChecksumLimit from './checksums'
 
-jest.mock(
-  'constants/config',
-  jest.fn(() => ({
-    chunkedChecksums: true,
-  })),
-)
+jest.mock('constants/config', () => ({}))
 
 describe('utils/checksums', () => {
-  describe('computeFileChecksumLimit, chunked', () => {
+  describe('computeFileChecksumLimit', () => {
     it('produces a correct checksum given an empty file', async () => {
       const file = new File([], 'empty')
       await expect(computeFileChecksumLimit(file)).resolves.toEqual({

catalog/app/utils/checksums/checksums.spec.ts

@@ -1,6 +1,5 @@
 import pLimit from 'p-limit'
 
-import cfg from 'constants/config'
 import * as Model from 'model'
 
 // 8 MiB -- boto3 default: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
@@ -9,7 +8,6 @@ export const MIN_PART_SIZE = 1024 ** 2 * 8
 const MAX_PARTS = 10000 // Maximum number of parts per upload supported by S3
 
 export function getPartSize(fileSize: number): number | null {
-  // use single-part upload (and plain SHA256 hash)
   if (fileSize < MIN_PART_SIZE) return null
 
   // NOTE: in the case where fileSize is exactly equal to MIN_PART_SIZE
@@ -25,12 +23,7 @@ export function getPartSize(fileSize: number): number | null {
   return partSize
 }
 
-const plain = (value: ArrayBuffer): Model.Checksum => ({
-  value: Buffer.from(value).toString('hex'),
-  type: Model.CHECKSUM_TYPE_SHA256,
-})
-
-const chunked = (value: ArrayBuffer): Model.Checksum => ({
+const sha256Chunked = (value: ArrayBuffer): Model.Checksum => ({
   value: Buffer.from(value).toString('base64'),
   type: Model.CHECKSUM_TYPE_SHA256_CHUNKED,
 })
@@ -60,8 +53,6 @@ const hashBlobLimit = (blob: Blob) => blobLimit(hashBlob, blob)
 async function computeFileChecksum(f: File): Promise<Model.Checksum> {
   if (!crypto?.subtle?.digest) throw new Error('Crypto API unavailable')
 
-  if (!cfg.chunkedChecksums) return plain(await hashBlobLimit(f))
-
   const partSize = getPartSize(f.size) ?? f.size
   const parts: Blob[] = []
 
@@ -74,7 +65,7 @@ async function computeFileChecksum(f: File): Promise<Model.Checksum> {
 
   const checksums = await Promise.all(parts.map(hashBlobLimit))
   const value = await crypto.subtle.digest('SHA-256', mergeBuffers(checksums))
-  return chunked(value)
+  return sha256Chunked(value)
 }
 
 const computeFileChecksumLimit = (f: File) => fileLimit(computeFileChecksum, f)

catalog/app/utils/checksums/checksums.ts

@@ -85,9 +85,9 @@
     "desktop": {
       "type": "boolean"
     },
-    "chunkedChecksums": {
+    "crc64Checksums": {
       "type": "boolean",
-      "description": "Whether to use chunked checksums when creating / modifying packages via the Catalog UI."
+      "description": "Whether to use CRC64/NVME checksums when creating / modifying packages via the Catalog UI."
     },
     "build_version": {
       "type": "string",

catalog/config-schema.json

@@ -14,7 +14,7 @@
     "analyticsBucket": "${ANALYTICS_BUCKET}",
     "serviceBucket": "${SERVICE_BUCKET}",
     "mode": "${CATALOG_MODE}",
-    "chunkedChecksums": ${CHUNKED_CHECKSUMS},
+    "crc64Checksums": ${CRC64_CHECKSUMS},
     "qurator": ${QURATOR},
     "stackVersion": "${STACK_VERSION}",
     "packageRoot": "${PACKAGE_ROOT}"