Improvement/cldsrv 724 versionning related functional tests

package.json

@@ -19,11 +19,17 @@
   },
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.908.0",
+    "@aws-sdk/credential-providers": "^3.864.0",
+    "@aws-sdk/middleware-retry": "^3.374.0",
+    "@aws-sdk/protocol-http": "^3.374.0",
+    "@aws-sdk/s3-request-presigner": "^3.901.0",
+    "@aws-sdk/signature-v4": "^3.374.0",
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
+    "@smithy/node-http-handler": "^3.0.0",
     "arsenal": "git+https://github.com/scality/Arsenal#8.2.41",
     "async": "2.6.4",
-    "aws-sdk": "^2.1692.0",
     "bucketclient": "scality/bucketclient#8.2.7",
     "bufferutil": "^4.0.8",
     "commander": "^12.1.0",

tests/functional/aws-node-sdk/lib/utility/bucket-util.js

@@ -1,151 +1,146 @@
-const AWS = require('aws-sdk');
-AWS.config.logger = console;
-const { S3 } = require('aws-sdk');
+const {
+    S3Client,
+    HeadBucketCommand,
+    CreateBucketCommand,
+    DeleteBucketCommand,
+    ListObjectVersionsCommand,
+    DeleteObjectCommand,
+    ListBucketsCommand,
+} = require('@aws-sdk/client-s3');
 const projectFixture = require('../fixtures/project');
 const getConfig = require('../../test/support/config');
 
 class BucketUtility {
-    constructor(profile = 'default', config = {}) {
+    constructor(profile = 'default', config = {}, unauthenticated = false) {
         const s3Config = getConfig(profile, config);
-
-        this.s3 = new S3(s3Config);
-        this.s3.config.setPromisesDependency(Promise);
-        this.s3.config.update({
-            maxRetries: 0,
-        });
+        if (unauthenticated) {
+            this.s3 = new S3Client({
+                ...s3Config,
+                maxAttempts: 0,
+                credentials: { accessKeyId: '', secretAccessKey: '' },
+                forcePathStyle: true,
+                signer: { sign: async request => request },
+             });
+        }
+       else {
+            this.s3 = new S3Client({
+                ...s3Config,
+                maxAttempts: 0,
+             });
+       }
     }
 
     bucketExists(bucketName) {
-        return this.s3
-            .headBucket({ Bucket: bucketName })
-            .promise()
+        return this.s3.send(new HeadBucketCommand({ Bucket: bucketName }))
             .then(() => true)
             .catch(err => {
-                if (err.code === 'NotFound') {
+                if (err.name === 'NotFound') {
                     return false;
                 }
                 throw err;
             });
     }
 
     createOne(bucketName) {
-        return this.s3
-            .createBucket({ Bucket: bucketName })
-            .promise()
-            .then(() => bucketName);
+        return this.s3.send(new CreateBucketCommand({ Bucket: bucketName }))
+            .then(() => bucketName)
+            .catch(err => {
+                throw err;
+            });
     }
 
     createOneWithLock(bucketName) {
-        return this.s3
-            .createBucket({
-                Bucket: bucketName,
-                ObjectLockEnabledForBucket: true,
-            })
-            .promise()
-            .then(() => bucketName);
+        return this.s3.send(new CreateBucketCommand({
+            Bucket: bucketName,
+            ObjectLockEnabledForBucket: true,
+        })).then(() => bucketName);
     }
 
     createMany(bucketNames) {
         const promises = bucketNames.map(bucketName =>
             this.createOne(bucketName),
         );
-
         return Promise.all(promises);
     }
 
     createRandom(nBuckets = 1) {
         if (nBuckets === 1) {
             const bucketName = projectFixture.generateBucketName();
-
             return this.createOne(bucketName);
         }
-
         const bucketNames = projectFixture
             .generateManyBucketNames(nBuckets)
-            .sort(() => 0.5 - Math.random()); // Simply shuffle array
-
+            .sort(() => 0.5 - Math.random());
         return this.createMany(bucketNames);
     }
 
     deleteOne(bucketName) {
-        return this.s3.deleteBucket({ Bucket: bucketName }).promise();
+        return this.s3.send(new DeleteBucketCommand({ Bucket: bucketName }));
     }
 
     deleteMany(bucketNames) {
         const promises = bucketNames.map(bucketName =>
             this.deleteOne(bucketName),
         );
-
         return Promise.all(promises);
     }
-
+    
     /**
      * Recursively delete all versions of all objects within the bucket
      * @param bucketName
      * @returns {Promise.<T>}
      */
-
-    async empty(bucketName, BypassGovernanceRetention = false) {
+    empty(bucketName, BypassGovernanceRetention = false) {
         const param = {
             Bucket: bucketName,
         };
 
-        const listedObjects = await this.s3.listObjectVersions(param).promise();
-
-        for (const version of listedObjects.Versions) {
-            if (version.Key.endsWith('/')) {
-                continue;
-            }
-
-            await this.s3
-                .deleteObject({
-                    Bucket: bucketName,
-                    Key: version.Key,
-                    VersionId: version.VersionId,
-                    ...(BypassGovernanceRetention && {
-                        BypassGovernanceRetention,
-                    }),
-                })
-                .promise();
-        }
-
-        for (const version of listedObjects.Versions) {
-            if (!version.Key.endsWith('/')) {
-                continue;
-            }
-
-            await this.s3
-                .deleteObject({
-                    Bucket: bucketName,
-                    Key: version.Key,
-                    VersionId: version.VersionId,
-                    ...(BypassGovernanceRetention && {
-                        BypassGovernanceRetention,
-                    }),
-                })
-                .promise();
-        }
-
-        for (const marker of listedObjects.DeleteMarkers) {
-            await this.s3
-                .deleteObject({
-                    Bucket: bucketName,
-                    Key: marker.Key,
-                    VersionId: marker.VersionId,
-                    ...(BypassGovernanceRetention && {
-                        BypassGovernanceRetention,
-                    }),
-                })
-                .promise();
-        }
+        return this.s3.send(new ListObjectVersionsCommand(param))
+            .then(data => Promise.all(
+                (data.Versions || [])
+                .filter(object => !object.Key.endsWith('/'))
+                .map(object =>
+                    this.s3.send(new DeleteObjectCommand({
+                        Bucket: bucketName,
+                        Key: object.Key,
+                        VersionId: object.VersionId,
+                        ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
+                    })).then(() => object)
+                )
+                .concat((data.Versions || [])
+                    .filter(object => object.Key.endsWith('/'))
+                    .map(object =>
+                        this.s3.send(new DeleteObjectCommand({
+                            Bucket: bucketName,
+                            Key: object.Key,
+                            VersionId: object.VersionId,
+                            ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
+                        }))
+                        .then(() => object)
+                    )
+                )
+                .concat((data.DeleteMarkers || [])
+                    .map(object =>
+                    this.s3.send(new DeleteObjectCommand({
+                        Bucket: bucketName,
+                        Key: object.Key,
+                        VersionId: object.VersionId,
+                        ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
+                        }))
+                        .then(() => object)
+                    )
+                )
+            )
+        );
     }
 
     emptyMany(bucketNames) {
-        const promises = bucketNames.map(bucketName => this.empty(bucketName));
-
+        const promises = bucketNames.map(
+            bucketName => this.empty(bucketName)
+        );
         return Promise.all(promises);
     }
-
+    
     emptyIfExists(bucketName) {
         return this.bucketExists(bucketName).then(exists => {
             if (exists) {
@@ -159,15 +154,15 @@ class BucketUtility {
         const promises = bucketNames.map(bucketName =>
             this.emptyIfExists(bucketName),
         );
-
         return Promise.all(promises);
     }
 
     getOwner() {
-        return this.s3
-            .listBuckets()
-            .promise()
-            .then(data => data.Owner);
+        return this.s3.send(new ListBucketsCommand({}))
+            .then(data => data.Owner)
+            .catch(err => {
+                throw err;
+            });
     }
 }
 

tests/functional/aws-node-sdk/lib/utility/checkError.js

@@ -2,8 +2,12 @@ const assert = require('assert');
 
 function checkError(err, code, statusCode) {
     assert(err, 'Expected error but found none');
-    assert.strictEqual(err.code, code);
-    assert.strictEqual(err.statusCode, statusCode);
+    if (code) {
+        assert.strictEqual(err.name, code);
+    }
+    if (statusCode) {
+        assert.strictEqual(err.$metadata.httpStatusCode, statusCode);
+    }
 }
 
 module.exports = checkError;

tests/functional/aws-node-sdk/lib/utility/customS3Request.js

@@ -1,43 +1,59 @@
-const { S3 } = require('aws-sdk');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { HttpRequest } = require('@smithy/protocol-http');
 const querystring = require('querystring');
 
 const getConfig = require('../../test/support/config');
 
-const config = getConfig('default', { signatureVersion: 'v4' });
-const s3 = new S3(config);
+const config = getConfig('default');
+
+
+// Custom middleware to modify requests without mutating args
+const customRequestMiddleware = buildParams => next => async args => {
 
-function customS3Request(action, params, buildParams, callback) {
-    const method = action.bind(s3);
-    const request = method(params);
     const { headers, query } = buildParams;
-    // modify underlying http request object created by aws sdk
-    request.on('build', () => {
-        Object.assign(request.httpRequest.headers, headers);
-        if (query) {
-            const qs = querystring.stringify(query);
-            // NOTE: that this relies on there not being a query string in the
-            // first place; if there is a qs then we have to search for ? and
-            // append &qs at the end of the string, if ? is not followed by ''
-            request.httpRequest.path = `${request.httpRequest.path}?${qs}`;
-        }
-    });
-    request.on('success', response => {
-        const resData = {
-            statusCode: response.httpResponse.statusCode,
-            headers: response.httpResponse.headers,
-            body: response.httpResponse.body.toString('utf8'),
-        };
-        callback(null, resData);
+
+    const prevReq = args.request;
+    const base = prevReq instanceof HttpRequest ? prevReq : new HttpRequest(prevReq);
+
+    let newHeaders = base.headers || {};
+    if (headers) {
+        newHeaders = { ...newHeaders, ...headers };
+    }
+
+    let newQuery = base.query || {};
+    if (query) {
+        const extra = querystring.parse(querystring.stringify(query));
+        newQuery = { ...newQuery, ...extra };
+    }
+
+    const newReq = new HttpRequest({
+        ...base,
+        headers: newHeaders,
+        query: newQuery,
     });
-    request.on('error', err => {
+
+    return next({ ...args, request: newReq });
+};
+
+async function customS3Request(CommandClass, params, buildParams) {
+        const customS3 = new S3Client({ ...config });
+
+        customS3.middlewareStack.add(
+            customRequestMiddleware(buildParams),
+            { step: 'build', name: 'customRequestMiddleware', tags: ['CUSTOM'] }
+        );
+
+        const command = new CommandClass(params);
+        const response = await customS3.send(command);
+
         const resData = {
-            statusCode: request.response.httpResponse.statusCode,
-            headers: request.response.httpResponse.headers,
-            body: request.response.httpResponse.body.toString('utf8'),
+            statusCode: 200,
+            headers: response.$metadata?.httpHeaders || {},
+            body: JSON.stringify(response),
         };
-        callback(err, resData);
-    });
-    request.send();
+
+        return resData;
+
 }
 
 module.exports = customS3Request;

tests/functional/aws-node-sdk/lib/utility/tagging.js

@@ -5,9 +5,11 @@ const taggingTests = [
         it: 'should return tags if value is an empty string' },
     { tag: { key: 'w'.repeat(129), value: 'foo' },
         error: 'InvalidTag',
+        code: 400,
         it: 'should return InvalidTag if key length is greater than 128' },
     { tag: { key: 'bar', value: 'f'.repeat(257) },
         error: 'InvalidTag',
+        code: 400,
         it: 'should return InvalidTag if key length is greater than 256',
     },
 ];