Improvement/s3 utils 191

CountItems/CountWorker.js

@@ -4,6 +4,7 @@ const { BucketInfo } = require('arsenal').models;
 const monitoring = require('../utils/monitoring');
 const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
 
+const PENSIEVE = 'PENSIEVE';
 class CountWorker {
     constructor(params) {
         this.log = params.log;
@@ -27,6 +28,40 @@ class CountWorker {
         return this.client.setup(callback);
     }
 
+    getIsTransient(bucketInfo, cb) {
+        const locConstraint = bucketInfo.getLocationConstraint();
+
+        if (this.client.isLocationTransient) {
+            this.client.isLocationTransient(locConstraint, this.log, cb);
+            return;
+        }
+        this.pensieveLocationIsTransient(locConstraint, cb);
+    }
+
+    pensieveLocationIsTransient(locConstraint, cb) {
+        const overlayVersionId = 'configuration/overlay-version';
+
+        async.waterfall([
+            next => this.client.getObject(PENSIEVE, overlayVersionId, null, this.log, next),
+            (version, next) => {
+                const overlayConfigId = `configuration/overlay/${version}`;
+                return this.client.getObject(PENSIEVE, overlayConfigId, null, this.log, next);
+            },
+        ], (err, res) => {
+            if (err) {
+                this.log.error('error getting configuration overlay', {
+                    method: 'pensieveLocationIsTransient',
+                    error: err,
+                });
+                return cb(err);
+            }
+            const isTransient =
+                Boolean(res?.locations[locConstraint]?.isTransient);
+
+            return cb(null, isTransient);
+        });
+    }
+
     countItems(bucketInfoObj, callback) {
         if (!this.client.client) {
             return callback(new Error('NotConnected'));
@@ -40,7 +75,7 @@ class CountWorker {
         const bucketName = bucketInfo.getName();
         this.log.info(`${process.pid} handling ${bucketName}`);
         return async.waterfall([
-            next => this.client._getIsTransient(bucketInfo, this.log, next),
+            next => this.getIsTransient(bucketInfo, next),
             (isTransient, next) => this.client.getObjectMDStats(bucketName, bucketInfo, isTransient, this.log, next),
         ], (err, results) => {
             monitoring.bucketsCount.inc({ status: err ? 'error' : 'success' });

package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3utils",
-  "version": "1.15.9",
+  "version": "1.16.0",
   "engines": {
     "node": ">= 22"
   },
@@ -28,7 +28,7 @@
   "dependencies": {
     "@senx/warp10": "^2.0.3",
     "JSONStream": "^1.3.5",
-    "arsenal": "git+https://github.com/scality/arsenal#8.2.8",
+    "arsenal": "git+https://github.com/scality/arsenal#8.2.19",
     "async": "^3.2.6",
     "aws-sdk": "^2.1691.0",
     "bucketclient": "scality/bucketclient#8.2.2",

tests/mocks/mongoClient.js

@@ -4,8 +4,9 @@ module.exports = {
     },
     setup: jest.fn(),
     close: jest.fn(),
-    _getIsTransient: jest.fn(),
     getObjectMDStats: jest.fn(),
+    isLocationTransient: jest.fn(),
+    getObject: jest.fn(),
     getBucketInfos: jest.fn(),
     updateStorageConsumptionMetrics: jest.fn(),
     getUsersBucketCreationDate: jest.fn(),

tests/unit/CountItems/CountWorker.js

@@ -10,8 +10,8 @@ describe('CountItems::CountWorker', () => {
         mongoMock.close.mockReset();
         mongoMock.setup.mockReset();
         mongoMock.client.isConnected.mockReset();
-        mongoMock._getIsTransient.mockReset();
         mongoMock.getObjectMDStats.mockReset();
+        mongoMock.getObject.mockReset();
     });
 
     const t = [
@@ -180,13 +180,11 @@ describe('CountItems::CountWorker', () => {
             sendFn: testSendFn,
             client: mongoMock,
         });
+        w.getIsTransient = jest.fn((bucketInfo, cb) => cb(...tc.mock.getIsTransient));
         mongoMock.setup.mockImplementationOnce(cb => cb(...tc.mock.setup));
         mongoMock.close.mockImplementationOnce(cb => cb(...tc.mock.close));
         mongoMock.client.isConnected
             .mockImplementationOnce(() => tc.mock.isConnected);
-        mongoMock._getIsTransient.mockImplementationOnce(
-            (_a, _b, cb) => cb(...tc.mock.getIsTransient),
-        );
         mongoMock.getObjectMDStats.mockImplementationOnce(
             (_a, _b, _c, _d, cb) => cb(...tc.mock.getObjectMDStats),
         );
@@ -213,15 +211,212 @@ describe('CountItems::CountWorker', () => {
             _creationDate: Date.now().toString(),
             website: { indexDocument: 'index.html' },
         };
+        w.getIsTransient = jest.fn((bucketInfo, cb) => cb(null, true));
         mongoMock.setup.mockImplementationOnce(cb => cb());
         mongoMock.close.mockImplementationOnce(cb => cb());
         mongoMock.client.isConnected.mockImplementationOnce(() => false);
-        mongoMock._getIsTransient.mockImplementationOnce((_a, _b, cb) => cb(null, true));
         mongoMock.getObjectMDStats.mockImplementationOnce((_a, _b, _c, _d, cb) => cb(null, { value: 42 }));
         w.countItems(bucketInfo, (err, results) => {
             expect(err).toBeNull();
             expect(results).toEqual({ value: 42 });
             done();
         });
     });
+
+    describe('CountWorker.getIsTransient method', () => {
+        let worker;
+        let mockBucketInfo;
+
+        beforeEach(() => {
+            worker = new CountWorker({
+                log: new DummyLogger(),
+                sendFn: jest.fn(),
+                client: mongoMock,
+            });
+            mockBucketInfo = {
+                _name: 'test-bucket',
+                _owner: 'any',
+                _ownerDisplayName: 'any',
+                _creationDate: Date.now().toString(),
+                getLocationConstraint: jest.fn(() => 'test-location'),
+            };
+        });
+
+        test('should use client.isLocationTransient if available and return true', done => {
+            mongoMock.isLocationTransient = jest.fn((loc, lg, cb) => cb(null, true));
+            worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(true);
+                expect(mongoMock.isLocationTransient).toHaveBeenCalledWith('test-location', worker.log, expect.any(Function));
+                done();
+            });
+        });
+
+        test('should use client.isLocationTransient if available and return false', done => {
+            mongoMock.isLocationTransient = jest.fn((loc, lg, cb) => cb(null, false));
+            worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(false);
+                done();
+            });
+        });
+
+        test('should propagate error from client.isLocationTransient', done => {
+            const testError = new Error('client.isLocationTransient error');
+            mongoMock.isLocationTransient = jest.fn((loc, lg, cb) => cb(testError));
+            worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                expect(err).toBe(testError);
+                expect(isTransient).toBeUndefined();
+                done();
+            });
+        });
+
+        describe('when client.isLocationTransient is not available (fallback to pensieve)', () => {
+            beforeEach(() => {
+                mongoMock.isLocationTransient = undefined;
+            });
+
+            test('should fallback to pensieveLocationIsTransient and return true', done => {
+                mongoMock.getObject
+                    .mockImplementationOnce((bucket, key, params, log, cb) => cb(null, 'v1')) // 1st call
+                    .mockImplementationOnce((bucket, key, params, log, cb) => cb(null, { // 2nd call
+                        locations: { 'test-location': { isTransient: true } },
+                    }));
+                worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                    expect(err).toBeNull();
+                    expect(isTransient).toBe(true);
+                    expect(mongoMock.getObject).toHaveBeenCalledTimes(2);
+                    done();
+                });
+            });
+
+            test('should fallback to pensieveLocationIsTransient and return false', done => {
+                mongoMock.getObject
+                    .mockImplementationOnce((bucket, key, params, log, cb) => cb(null, 'v1'))
+                    .mockImplementationOnce((bucket, key, params, log, cb) => cb(null, {
+                        locations: { 'test-location': { isTransient: false } },
+                    }));
+                worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                    expect(err).toBeNull();
+                    expect(isTransient).toBe(false);
+                    done();
+                });
+            });
+
+            test('should propagate error from pensieveLocationIsTransient (e.g., first getObject fails)', done => {
+                const pensieveError = new Error('Pensieve getObject error');
+                mongoMock.getObject.mockImplementationOnce((b, k, p, l, cb) => cb(pensieveError));
+                worker.getIsTransient(mockBucketInfo, (err, isTransient) => {
+                    expect(err).toBe(pensieveError);
+                    expect(isTransient).toBeUndefined();
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('CountWorker.pensieveLocationIsTransient method', () => {
+        let worker;
+        const PENSIEVE_BUCKET_NAME = 'PENSIEVE';
+
+        beforeEach(() => {
+            worker = new CountWorker({
+                log: new DummyLogger(),
+                sendFn: jest.fn(),
+                client: mongoMock,
+            });
+        });
+
+        test('should return true if location is transient in Pensieve config', done => {
+            mongoMock.getObject
+                .mockImplementationOnce((bucket, key, params, log, cb) => {
+                    expect(bucket).toBe(PENSIEVE_BUCKET_NAME);
+                    expect(key).toBe('configuration/overlay-version');
+                    cb(null, 'v-p-1');
+                })
+                .mockImplementationOnce((bucket, key, params, log, cb) => {
+                    expect(bucket).toBe(PENSIEVE_BUCKET_NAME);
+                    expect(key).toBe('configuration/overlay/v-p-1');
+                    cb(null, { locations: { 'loc-A': { isTransient: true } } });
+                });
+
+            worker.pensieveLocationIsTransient('loc-A', (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(true);
+                done();
+            });
+        });
+
+        test('should return false if location is not transient in Pensieve config', done => {
+            mongoMock.getObject
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, 'v-p-2'))
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, { locations: { 'loc-B': { isTransient: false } } }));
+
+            worker.pensieveLocationIsTransient('loc-B', (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(false);
+                done();
+            });
+        });
+
+        test('should return false if isTransient property is missing', done => {
+            mongoMock.getObject
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, 'v-p-3'))
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, { locations: { 'loc-C': {} } }));
+
+            worker.pensieveLocationIsTransient('loc-C', (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(false);
+                done();
+            });
+        });
+
+        test('should propagate error from first getObject call', done => {
+            const testError = new Error('getObject overlay-version failed');
+            mongoMock.getObject.mockImplementationOnce((b, k, p, l, cb) => cb(testError));
+
+            worker.pensieveLocationIsTransient('loc-D', (err, isTransient) => {
+                expect(err).toBe(testError);
+                expect(isTransient).toBeUndefined();
+                done();
+            });
+        });
+
+        test('should propagate error from second getObject call', done => {
+            const testError = new Error('getObject overlay-config failed');
+            mongoMock.getObject
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, 'v-p-4'))
+                .mockImplementationOnce((b, k, p, l, cb) => cb(testError));
+
+            worker.pensieveLocationIsTransient('loc-E', (err, isTransient) => {
+                expect(err).toBe(testError);
+                expect(isTransient).toBeUndefined();
+                done();
+            });
+        });
+
+        test('should handle case when locConstraint is not in locations', done => {
+            mongoMock.getObject
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, 'v-p-5'))
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, { locations: { 'other-loc': { isTransient: true } } }));
+
+            worker.pensieveLocationIsTransient('loc-F-not-in-response', (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(false);
+                done();
+            });
+        });
+
+        test('should handle case where no location is provided', done => {
+            mongoMock.getObject
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, 'v-p-6'))
+                .mockImplementationOnce((b, k, p, l, cb) => cb(null, { locations: {} }));
+
+            worker.pensieveLocationIsTransient('loc-G', (err, isTransient) => {
+                expect(err).toBeNull();
+                expect(isTransient).toBe(false);
+                done();
+            });
+        });
+    });
 });