ARSN-557: add checksum to object metadata

Author: leif-scality

lib/models/ObjectMD.ts

@@ -9,6 +9,7 @@ import ObjectMDLocation, {
 import ObjectMDAmzRestore from './ObjectMDAmzRestore';
 import ObjectMDArchive from './ObjectMDArchive';
 import { ObjectMDAzureInfoMetadata } from './ObjectMDAzureInfo';
+import ObjectMDChecksum, { ChecksumAlgorithm, ChecksumType } from './ObjectMDChecksum';
 
 export type ACL = {
     Canned: string;
@@ -102,6 +103,7 @@ export type ObjectMDData = {
     // This is the canonical Id for the bucket owner.
     // This is only set when it differs from `owner-id`.
     bucketOwnerId?: string;
+    checksum?: ObjectMDChecksum;
 };
 
 /**
@@ -275,6 +277,10 @@ export default class ObjectMD {
             // @ts-ignore
             this.setLocation([{ key: this._data.location }]);
         }
+        if (this._data.checksum && !(this._data.checksum instanceof ObjectMDChecksum)) {
+            const { checksumAlgorithm, checksumValue, checksumType } = this._data.checksum;
+            this._data.checksum = new ObjectMDChecksum(checksumAlgorithm, checksumValue, checksumType);
+        }
     }
 
     /**
@@ -477,6 +483,38 @@ export default class ObjectMD {
         return this._data['content-md5'];
     }
 
+    /**
+     * Set checksum
+     *
+     * @param checksum - ObjectMDChecksum instance
+     * @return itself
+     */
+    setChecksum(checksum: ObjectMDChecksum | {
+        checksumAlgorithm: ChecksumAlgorithm;
+        checksumValue: string;
+        checksumType: ChecksumType;
+    }) {
+        if (checksum instanceof ObjectMDChecksum) {
+            this._data.checksum = checksum;
+        } else if (ObjectMDChecksum.isValid(checksum) === null) {
+            this._data.checksum = new ObjectMDChecksum(
+                checksum.checksumAlgorithm,
+                checksum.checksumValue,
+                checksum.checksumType,
+            );
+        } else {
+            throw new Error('checksum must be of type ObjectMDChecksum.');
+        }
+        return this;
+    }
+
+    /**
+     * Returns checksum as an ObjectMDChecksum instance, or null if not set.
+     */
+    getChecksum(): ObjectMDChecksum | null {
+        return this._data.checksum ?? null;
+    }
+
     /**
      * Set content-language
      *

lib/models/ObjectMDChecksum.ts

@@ -0,0 +1,83 @@
+export const CHECKSUM_ALGORITHMS = ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256'] as const;
+export const CHECKSUM_TYPES = ['FULL_OBJECT', 'COMPOSITE'] as const;
+
+export type ChecksumAlgorithm = typeof CHECKSUM_ALGORITHMS[number];
+export type ChecksumType = typeof CHECKSUM_TYPES[number];
+
+const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
+
+type AlgoSpec = {
+    xmlTag: string;
+    digestLength: number;
+};
+
+const algoSpecs: Record<ChecksumAlgorithm, AlgoSpec> = {
+    crc32:     { xmlTag: 'ChecksumCRC32',    digestLength: 8  },
+    crc32c:    { xmlTag: 'ChecksumCRC32C',   digestLength: 8  },
+    crc64nvme: { xmlTag: 'ChecksumCRC64NVME', digestLength: 12 },
+    sha1:      { xmlTag: 'ChecksumSHA1',     digestLength: 28 },
+    sha256:    { xmlTag: 'ChecksumSHA256',   digestLength: 44 },
+};
+
+function isValidDigest(algorithm: ChecksumAlgorithm, value: string): boolean {
+    const { digestLength } = algoSpecs[algorithm];
+    return typeof value === 'string' && value.length === digestLength && base64Regex.test(value);
+}
+
+/**
+ * Represents an object checksum stored in object metadata.
+ *
+ * Internal representation uses plain algorithm/value/type fields.
+ * The toGetObjectAttributesXML() method produces the wire XML fragment
+ * expected inside a GetObjectAttributesResponse Checksum element.
+ */
+export default class ObjectMDChecksum {
+    checksumAlgorithm: ChecksumAlgorithm;
+    checksumValue: string;
+    checksumType: ChecksumType;
+
+    static isValid(data: {
+        checksumAlgorithm: ChecksumAlgorithm;
+        checksumValue: string;
+        checksumType: ChecksumType;
+    }): string | null {
+        if (!CHECKSUM_ALGORITHMS.includes(data.checksumAlgorithm)) {
+            return `invalid checksumAlgorithm: ${data.checksumAlgorithm}`;
+        }
+        if (!CHECKSUM_TYPES.includes(data.checksumType)) {
+            return `invalid checksumType: ${data.checksumType}`;
+        }
+        if (!isValidDigest(data.checksumAlgorithm, data.checksumValue)) {
+            return `invalid checksumValue for ${data.checksumAlgorithm}: ${data.checksumValue}`;
+        }
+        return null;
+    }
+
+    constructor(
+        checksumAlgorithm: ChecksumAlgorithm,
+        checksumValue: string,
+        checksumType: ChecksumType,
+    ) {
+        const error = ObjectMDChecksum.isValid({ checksumAlgorithm, checksumValue, checksumType });
+        if (error !== null) {
+            throw new Error(error);
+        }
+        this.checksumAlgorithm = checksumAlgorithm;
+        this.checksumValue = checksumValue;
+        this.checksumType = checksumType;
+    }
+
+    /**
+     * Returns the XML fragment for the Checksum element in a
+     * GetObjectAttributes response, e.g.:
+     *   <ChecksumSHA256>abc=</ChecksumSHA256>
+     *   <ChecksumType>FULL_OBJECT</ChecksumType>
+     */
+    toGetObjectAttributesXML(): string {
+        const { xmlTag } = algoSpecs[this.checksumAlgorithm];
+        return '<Checksum>' +
+            `<${xmlTag}>${this.checksumValue}</${xmlTag}>` +
+            `<ChecksumType>${this.checksumType}</ChecksumType>` +
+            '</Checksum>';
+    }
+}

lib/models/index.ts

@@ -9,6 +9,8 @@ export { default as NotificationConfiguration } from './NotificationConfiguratio
 export { default as ObjectLockConfiguration } from './ObjectLockConfiguration';
 export { default as ObjectMD } from './ObjectMD';
 export { default as ObjectMDAmzRestore } from './ObjectMDAmzRestore';
+export { default as ObjectMDChecksum, CHECKSUM_ALGORITHMS, CHECKSUM_TYPES } from './ObjectMDChecksum';
+export type { ChecksumAlgorithm, ChecksumType } from './ObjectMDChecksum';
 export { default as ObjectMDArchive } from './ObjectMDArchive';
 export { default as ObjectMDAzureInfo } from './ObjectMDAzureInfo';
 export { default as ObjectMDLocation } from './ObjectMDLocation';

tests/unit/models/ObjectMD.spec.js

@@ -1,5 +1,6 @@
 const assert = require('assert');
 const ObjectMD = require('../../../lib/models/ObjectMD').default;
+const ObjectMDChecksum = require('../../../lib/models/ObjectMDChecksum').default;
 const constants = require('../../../lib/constants');
 const ExternalNullVersionId = require('../../../lib/versioning/constants')
     .VersioningConstants.ExternalNullVersionId;
@@ -860,3 +861,72 @@ describe('ObjectMD::getEncodedVersionId', () => {
         assert.strictEqual(objMd.getEncodedVersionId(), ExternalNullVersionId);
     });
 });
+
+describe('ObjectMD checksum', () => {
+    const sha256Digest = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=';
+
+    it('should store an ObjectMDChecksum instance when setChecksum is called with a valid plain object', () => {
+        const md = new ObjectMD();
+        md.setChecksum({
+            checksumAlgorithm: 'sha256',
+            checksumValue: sha256Digest,
+            checksumType: 'FULL_OBJECT',
+        });
+        const c = md.getChecksum();
+        assert(c instanceof ObjectMDChecksum);
+        assert.doesNotThrow(() => c.toGetObjectAttributesXML());
+    });
+
+    it('should throw when setChecksum is given an invalid object', () => {
+        const md = new ObjectMD();
+        assert.throws(() => {
+            md.setChecksum({ checksumAlgorithm: 'sha256' });
+        }, /ObjectMDChecksum/);
+    });
+
+    it('should return null when no checksum is set', () => {
+        const md = new ObjectMD();
+        assert.strictEqual(md.getChecksum(), null);
+    });
+
+    it('should preserve algorithm, value, and type through setChecksum / getChecksum', () => {
+        const md = new ObjectMD();
+        md.setChecksum(new ObjectMDChecksum('crc64nvme', 'HyOpGHolkII=', 'FULL_OBJECT'));
+        const result = md.getChecksum();
+        assert(result !== null);
+        assert.strictEqual(result.checksumAlgorithm, 'crc64nvme');
+        assert.strictEqual(result.checksumValue, 'HyOpGHolkII=');
+        assert.strictEqual(result.checksumType, 'FULL_OBJECT');
+    });
+
+    it('should return an ObjectMDChecksum instance after JSON round-trip', () => {
+        const md = new ObjectMD();
+        md.setChecksum(new ObjectMDChecksum('sha256', sha256Digest, 'FULL_OBJECT'));
+        const { result } = ObjectMD.createFromBlob(md.getSerialized());
+        assert(result !== undefined);
+        assert(result.getChecksum() instanceof ObjectMDChecksum);
+    });
+
+    it('should preserve algorithm, value, and type through JSON round-trip', () => {
+        const md = new ObjectMD();
+        md.setChecksum(new ObjectMDChecksum('sha256', sha256Digest, 'COMPOSITE'));
+        const { result } = ObjectMD.createFromBlob(md.getSerialized());
+        assert(result !== undefined);
+        const c = result.getChecksum();
+        assert.strictEqual(c.checksumAlgorithm, 'sha256');
+        assert.strictEqual(c.checksumValue, sha256Digest);
+        assert.strictEqual(c.checksumType, 'COMPOSITE');
+    });
+
+    it('should produce valid XML from getChecksum after JSON round-trip', () => {
+        const md = new ObjectMD();
+        md.setChecksum(new ObjectMDChecksum('crc64nvme', 'HyOpGHolkII=', 'FULL_OBJECT'));
+        const { result } = ObjectMD.createFromBlob(md.getSerialized());
+        assert(result !== undefined);
+        const xml = result.getChecksum().toGetObjectAttributesXML();
+        assert(xml.startsWith('<Checksum>'));
+        assert(xml.includes('<ChecksumCRC64NVME>HyOpGHolkII=</ChecksumCRC64NVME>'));
+        assert(xml.includes('<ChecksumType>FULL_OBJECT</ChecksumType>'));
+        assert(xml.endsWith('</Checksum>'));
+    });
+});

tests/unit/models/ObjectMDChecksum.spec.ts

@@ -0,0 +1,132 @@
+import assert from 'assert';
+
+import ObjectMDChecksum, { ChecksumAlgorithm } from '../../../lib/models/ObjectMDChecksum';
+
+// Valid base64 digest values of the correct length for each algorithm.
+const validDigest: Record<ChecksumAlgorithm, string> = {
+    crc32:     'AAAAAA==',                                           // 8 chars
+    crc32c:    'AAAAAA==',                                           // 8 chars
+    crc64nvme: 'AAAAAAAAAAA=',                                       // 12 chars
+    sha1:      'AAAAAAAAAAAAAAAAAAAAAAAAAAA=',                       // 28 chars
+    sha256:    'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=',       // 44 chars
+};
+
+describe('ObjectMDChecksum', () => {
+    describe('constructor', () => {
+        it('should store algorithm, value, and type', () => {
+            const c = new ObjectMDChecksum('sha256', validDigest.sha256, 'FULL_OBJECT');
+            assert.strictEqual(c.checksumAlgorithm, 'sha256');
+            assert.strictEqual(c.checksumValue, validDigest.sha256);
+            assert.strictEqual(c.checksumType, 'FULL_OBJECT');
+        });
+
+        it('should throw on invalid checksumAlgorithm', () => {
+            assert.throws(
+                // @ts-expect-error intentionally invalid
+                () => new ObjectMDChecksum('sha999', validDigest.sha256, 'FULL_OBJECT'),
+                /invalid checksumAlgorithm/,
+            );
+        });
+
+        it('should throw on invalid checksumType', () => {
+            assert.throws(
+                // @ts-expect-error intentionally invalid
+                () => new ObjectMDChecksum('sha256', validDigest.sha256, 'WRONG'),
+                /invalid checksumType/,
+            );
+        });
+
+        it('should throw on invalid checksumValue (wrong length)', () => {
+            assert.throws(
+                () => new ObjectMDChecksum('sha256', 'tooshort=', 'FULL_OBJECT'),
+                /invalid checksumValue/,
+            );
+        });
+
+        it('should throw on invalid checksumValue (not base64)', () => {
+            assert.throws(
+                () => new ObjectMDChecksum('sha256', '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!=', 'FULL_OBJECT'),
+                /invalid checksumValue/,
+            );
+        });
+    });
+
+    describe('toGetObjectAttributesXML', () => {
+        const cases: [ChecksumAlgorithm, string][] = [
+            ['crc32',     'ChecksumCRC32'],
+            ['crc32c',    'ChecksumCRC32C'],
+            ['crc64nvme', 'ChecksumCRC64NVME'],
+            ['sha1',      'ChecksumSHA1'],
+            ['sha256',    'ChecksumSHA256'],
+        ];
+
+        for (const [algo, xmlTag] of cases) {
+            it(`should use <${xmlTag}> for algorithm "${algo}"`, () => {
+                const c = new ObjectMDChecksum(algo, validDigest[algo], 'FULL_OBJECT');
+                assert(c.toGetObjectAttributesXML().includes(`<${xmlTag}>`),
+                    `expected <${xmlTag}> in XML`);
+            });
+        }
+
+        it('should wrap the checksum value inside the algorithm tag', () => {
+            const c = new ObjectMDChecksum('crc64nvme', 'HyOpGHolkII=', 'FULL_OBJECT');
+            assert(c.toGetObjectAttributesXML().includes('<ChecksumCRC64NVME>HyOpGHolkII=</ChecksumCRC64NVME>'));
+        });
+
+        it('should wrap everything in a <Checksum> element', () => {
+            const c = new ObjectMDChecksum('sha256', validDigest.sha256, 'FULL_OBJECT');
+            const xml = c.toGetObjectAttributesXML();
+            assert(xml.startsWith('<Checksum>'));
+            assert(xml.endsWith('</Checksum>'));
+        });
+
+        it('should append <ChecksumType>FULL_OBJECT</ChecksumType>', () => {
+            const c = new ObjectMDChecksum('sha256', validDigest.sha256, 'FULL_OBJECT');
+            assert(c.toGetObjectAttributesXML().includes('<ChecksumType>FULL_OBJECT</ChecksumType>'));
+        });
+
+        it('should append <ChecksumType>COMPOSITE</ChecksumType> for composite type', () => {
+            const c = new ObjectMDChecksum('crc32', validDigest.crc32, 'COMPOSITE');
+            assert(c.toGetObjectAttributesXML().includes('<ChecksumType>COMPOSITE</ChecksumType>'));
+        });
+    });
+
+    describe('isValid', () => {
+        it('should return null for a valid checksum object', () => {
+            assert.strictEqual(ObjectMDChecksum.isValid({
+                checksumAlgorithm: 'sha256',
+                checksumValue: validDigest.sha256,
+                checksumType: 'FULL_OBJECT',
+            }), null);
+        });
+
+        it('should return an error string for an invalid checksumAlgorithm', () => {
+            const result = ObjectMDChecksum.isValid({
+                // @ts-expect-error intentionally invalid
+                checksumAlgorithm: 'sha999',
+                checksumValue: validDigest.sha256,
+                checksumType: 'FULL_OBJECT',
+            });
+            assert.match(result!, /invalid checksumAlgorithm/);
+        });
+
+        it('should return an error string for an invalid checksumType', () => {
+            const result = ObjectMDChecksum.isValid({
+                checksumAlgorithm: 'sha256',
+                checksumValue: validDigest.sha256,
+                // @ts-expect-error intentionally invalid
+                checksumType: 'WRONG',
+            });
+            assert.match(result!, /invalid checksumType/);
+        });
+
+        it('should return an error string for an invalid checksumValue', () => {
+            const result = ObjectMDChecksum.isValid({
+                checksumAlgorithm: 'sha256',
+                checksumValue: 'tooshort=',
+                checksumType: 'FULL_OBJECT',
+            });
+            assert.match(result!, /invalid checksumValue/);
+        });
+    });
+});