Code and tests update

This commit actually implements the _getIsTransient
and _pensieveLocationIsTransient methods in the CountWorker.js previously
implemented on arsenal and only used by the CountWorker.js.
Issue: S3UTILS-191

Author: benzekrimaha

CountItems/CountWorker.js

@@ -4,6 +4,7 @@ const { BucketInfo } = require('arsenal').models;
 const monitoring = require('../utils/monitoring');
 const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
 
+const PENSIEVE = 'PENSIEVE';
 class CountWorker {
     constructor(params) {
         this.log = params.log;
@@ -27,6 +28,40 @@ class CountWorker {
         return this.client.setup(callback);
     }
 
+    getIsTransient(bucketInfo, cb) {
+        const locConstraint = bucketInfo.getLocationConstraint();
+
+        if (this.client.isLocationTransient) {
+            this.client.isLocationTransient(locConstraint, this.log, cb);
+            return;
+        }
+        this.pensieveLocationIsTransient(locConstraint, cb);
+    }
+
+    pensieveLocationIsTransient(locConstraint, cb) {
+        const overlayVersionId = 'configuration/overlay-version';
+
+        async.waterfall([
+            next => this.client.getObject(PENSIEVE, overlayVersionId, null, this.log, next),
+            (version, next) => {
+                const overlayConfigId = `configuration/overlay/${version}`;
+                return this.client.getObject(PENSIEVE, overlayConfigId, null, this.log, next);
+            },
+        ], (err, res) => {
+            if (err) {
+                this.log.error('error getting configuration overlay', {
+                    method: '_getIsTransient',
+                    error: err,
+                });
+                return cb(err);
+            }
+            const isTransient =
+                Boolean(res.locations[locConstraint].isTransient);
+
+            return cb(null, isTransient);
+        });
+    }
+
     countItems(bucketInfoObj, callback) {
         if (!this.client.client) {
             return callback(new Error('NotConnected'));
@@ -40,7 +75,7 @@ class CountWorker {
         const bucketName = bucketInfo.getName();
         this.log.info(`${process.pid} handling ${bucketName}`);
         return async.waterfall([
-            next => this.client._getIsTransient(bucketInfo, this.log, next),
+            next => this.getIsTransient(bucketInfo, next),
             (isTransient, next) => this.client.getObjectMDStats(bucketName, bucketInfo, isTransient, this.log, next),
         ], (err, results) => {
             monitoring.bucketsCount.inc({ status: err ? 'error' : 'success' });

tests/mocks/mongoClient.js

@@ -4,7 +4,6 @@ module.exports = {
     },
     setup: jest.fn(),
     close: jest.fn(),
-    _getIsTransient: jest.fn(),
     getObjectMDStats: jest.fn(),
     getBucketInfos: jest.fn(),
     updateStorageConsumptionMetrics: jest.fn(),

tests/unit/CountItems/CountWorker.js

@@ -10,7 +10,6 @@ describe('CountItems::CountWorker', () => {
         mongoMock.close.mockReset();
         mongoMock.setup.mockReset();
         mongoMock.client.isConnected.mockReset();
-        mongoMock._getIsTransient.mockReset();
         mongoMock.getObjectMDStats.mockReset();
     });
 
@@ -22,7 +21,6 @@ describe('CountItems::CountWorker', () => {
                     setup: [null],
                     close: [],
                     isConnected: false,
-                    getIsTransient: [],
                     getObjectMDStats: [],
                 },
                 incomingMessage: {
@@ -47,7 +45,6 @@ describe('CountItems::CountWorker', () => {
                     setup: [new Error('failed setup')],
                     close: [],
                     isConnected: false,
-                    getIsTransient: [],
                     getObjectMDStats: [],
                 },
                 incomingMessage: {
@@ -98,7 +95,6 @@ describe('CountItems::CountWorker', () => {
                     setup: [],
                     close: [new Error('failed teardown')],
                     isConnected: true,
-                    getIsTransient: [],
                     getObjectMDStats: [],
                 },
                 incomingMessage: {
@@ -124,7 +120,6 @@ describe('CountItems::CountWorker', () => {
                     setup: [],
                     close: [],
                     isConnected: true,
-                    getIsTransient: [null, true],
                     getObjectMDStats: [null, { value: 42 }],
                 },
                 incomingMessage: {
@@ -151,7 +146,6 @@ describe('CountItems::CountWorker', () => {
                     setup: [],
                     close: [],
                     isConnected: true,
-                    getIsTransient: [null, true],
                     getObjectMDStats: [new Error('count error')],
                 },
                 incomingMessage: {
@@ -180,13 +174,11 @@ describe('CountItems::CountWorker', () => {
             sendFn: testSendFn,
             client: mongoMock,
         });
+        w.getIsTransient = jest.fn((bucketInfo, cb) => cb(null, true));
         mongoMock.setup.mockImplementationOnce(cb => cb(...tc.mock.setup));
         mongoMock.close.mockImplementationOnce(cb => cb(...tc.mock.close));
         mongoMock.client.isConnected
             .mockImplementationOnce(() => tc.mock.isConnected);
-        mongoMock._getIsTransient.mockImplementationOnce(
-            (_a, _b, cb) => cb(...tc.mock.getIsTransient),
-        );
         mongoMock.getObjectMDStats.mockImplementationOnce(
             (_a, _b, _c, _d, cb) => cb(...tc.mock.getObjectMDStats),
         );
@@ -213,10 +205,10 @@ describe('CountItems::CountWorker', () => {
             _creationDate: Date.now().toString(),
             website: { indexDocument: 'index.html' },
         };
+        w.getIsTransient = jest.fn((bucketInfo, cb) => cb(null, true));
         mongoMock.setup.mockImplementationOnce(cb => cb());
         mongoMock.close.mockImplementationOnce(cb => cb());
         mongoMock.client.isConnected.mockImplementationOnce(() => false);
-        mongoMock._getIsTransient.mockImplementationOnce((_a, _b, cb) => cb(null, true));
         mongoMock.getObjectMDStats.mockImplementationOnce((_a, _b, _c, _d, cb) => cb(null, { value: 42 }));
         w.countItems(bucketInfo, (err, results) => {
             expect(err).toBeNull();