Skip deleted objects in lifecycle requeue batch instead of aborting

When replaying failed archive batches, a deleted object causes
ObjNotFound in handleBatch which aborts async.reduce and silently drops
all remaining objects. Log the error and continue processing.

Issue: BB-749

Author: maeldonn

extensions/lifecycle/tasks/LifecycleRequeueTask.js

@@ -140,11 +140,18 @@ class LifecycleRequeueTask extends BackbeatTask {
                             rest.try,
                             log,
                             (err, res) => {
-                                if (err) {
-                                    return nextObject(err);
+                                if (!err) {
+                                    return nextObject(null, res + objsPerBucket);
                                 }
 
-                                return nextObject(null, res + objsPerBucket);
+                                if (err.name === 'ObjNotFound') {
+                                    log.error('object not found, skipping', {
+                                        bucketName, objectKey, objectVersion,
+                                    });
+                                    return nextObject(null, objsPerBucket);
+                                }
+
+                                return nextObject(err);
                             }
                         ),
                     (err, res) => next(err, res)

tests/unit/lifecycle/LifecycleResetTransitionInProgressTask.spec.js

@@ -90,4 +90,88 @@ describe('LifecycleResetTransitionInProgressTask', () => {
             done();
         });
     });
+
+    describe('ObjNotFound handling in batch', () => {
+        const makeMd = () => new ObjectMD()
+            .setContentMd5('etag1')
+            .setTransitionInProgress(true)
+            .setUserMetadata({
+                'x-amz-meta-scal-s3-transition-attempt': 0,
+            });
+
+        it('should skip deleted objects and continue processing valid ones', done => {
+            const batchEntry = ActionQueueEntry.create('requeueTransition')
+                .setAttribute('target', {
+                    byAccount: {
+                        123: {
+                            bucket1: [
+                                { objectKey: 'obj1', objectVersion: 'v1', eTag: '"etag1"', try: 1 },
+                                { objectKey: 'obj2', objectVersion: 'v2', eTag: '"etag1"', try: 1 },
+                                { objectKey: 'obj3', objectVersion: 'v3', eTag: '"etag1"', try: 1 },
+                            ],
+                        },
+                    },
+                });
+
+            backbeatMetadataProxyClient.setMdObjForKey('obj1', makeMd());
+            backbeatMetadataProxyClient.setErrorForKey('obj2', { name: 'ObjNotFound' });
+            backbeatMetadataProxyClient.setMdObjForKey('obj3', makeMd());
+
+            task.processActionEntry(batchEntry, err => {
+                assert.ifError(err);
+                assert.deepStrictEqual(
+                    backbeatMetadataProxyClient._putCalls.sort(),
+                    ['obj1', 'obj3'],
+                );
+                done();
+            });
+        });
+
+        it('should complete without error when all objects are deleted', done => {
+            const batchEntry = ActionQueueEntry.create('requeueTransition')
+                .setAttribute('target', {
+                    byAccount: {
+                        123: {
+                            bucket1: [
+                                { objectKey: 'obj1', objectVersion: 'v1', eTag: '"etag1"', try: 1 },
+                                { objectKey: 'obj2', objectVersion: 'v2', eTag: '"etag1"', try: 1 },
+                            ],
+                        },
+                    },
+                });
+
+            backbeatMetadataProxyClient.setErrorForKey('obj1', { name: 'ObjNotFound' });
+            backbeatMetadataProxyClient.setErrorForKey('obj2', { name: 'ObjNotFound' });
+
+            task.processActionEntry(batchEntry, err => {
+                assert.ifError(err);
+                assert.deepStrictEqual(backbeatMetadataProxyClient._putCalls, []);
+                done();
+            });
+        });
+
+        it('should abort batch on non-ObjNotFound errors', done => {
+            const batchEntry = ActionQueueEntry.create('requeueTransition')
+                .setAttribute('target', {
+                    byAccount: {
+                        123: {
+                            bucket1: [
+                                { objectKey: 'obj1', objectVersion: 'v1', eTag: '"etag1"', try: 1 },
+                                { objectKey: 'obj2', objectVersion: 'v2', eTag: '"etag1"', try: 1 },
+                            ],
+                        },
+                    },
+                });
+
+            backbeatMetadataProxyClient.setErrorForKey('obj1', { name: 'InternalError' });
+            backbeatMetadataProxyClient.setMdObjForKey('obj2', makeMd());
+
+            task.processActionEntry(batchEntry, err => {
+                assert.ok(err);
+                assert.strictEqual(err.name, 'InternalError');
+                assert.deepStrictEqual(backbeatMetadataProxyClient._putCalls, []);
+                done();
+            });
+        });
+    });
 });

tests/unit/mocks.js

@@ -86,20 +86,37 @@ class BackbeatMetadataProxyMock {
         this.indexesObj = null;
         this.receivedIdxObj = null;
         this.error = null;
+        this._mdByKey = {};
+        this._errorByKey = {};
+        this._putCalls = [];
     }
 
     setMdObj(mdObj) {
         this.mdObj = mdObj;
     }
 
+    setMdObjForKey(objectKey, mdObj) {
+        this._mdByKey[objectKey] = mdObj;
+    }
+
+    setErrorForKey(objectKey, error) {
+        this._errorByKey[objectKey] = error;
+    }
+
     getMetadata(params, log, cb) {
+        const keyError = this._errorByKey[params.objectKey];
+        if (keyError) {
+            return cb(keyError);
+        }
         if (this.error) {
             return cb(this.error);
         }
-        return cb(null, { Body: this.mdObj.getSerialized() });
+        const md = this._mdByKey[params.objectKey] || this.mdObj;
+        return cb(null, { Body: md.getSerialized() });
     }
 
     putMetadata(params, log, cb) {
+        this._putCalls.push(params.objectKey);
         this.receivedMd = JSON.parse(params.mdBlob);
         this.mdObj = ObjectMD.createFromBlob(params.mdBlob).result;
         return cb();