feat(s3-bucket): add configurable S3Object deletion batch size, set 1000 as default

* feat: add configurable s3 object deletion batch size

* feat: update default batch size to 1000

Author: AMZN-bradhawk

pkg/awsmod/batch.go

@@ -15,7 +15,7 @@ const (
 	// DefaultBatchSize is the batch size we initialize when constructing a batch delete client.
 	// This value is used when calling DeleteObjects. This represents how many objects to delete
 	// per DeleteObjects call.
-	DefaultBatchSize = 100
+	DefaultBatchSize = 1000
 )
 
 // BatchError will contain the key and bucket of the object that failed to
@@ -227,12 +227,16 @@ type BatchDelete struct {
 //	}); err != nil {
 //		return err
 //	}
-func NewBatchDeleteWithClient(s3client DeleteObjectsAPIClient, options ...func(*BatchDelete)) *BatchDelete {
+func NewBatchDeleteWithClient(s3client DeleteObjectsAPIClient, batchSize int, options ...func(*BatchDelete)) *BatchDelete {
 	svc := &BatchDelete{
 		Client:    s3client,
 		BatchSize: DefaultBatchSize,
 	}
 
+	if batchSize != -1 {
+		svc.BatchSize = batchSize
+	}
+
 	for _, opt := range options {
 		opt(svc)
 	}
@@ -261,9 +265,10 @@ func NewBatchDeleteWithClient(s3client DeleteObjectsAPIClient, options ...func(*
 //	}); err != nil {
 //		return err
 //	}
-func NewBatchDelete(c *aws.Config, options ...func(*BatchDelete)) *BatchDelete {
+func NewBatchDelete(c *aws.Config, batchSize int, options ...func(*BatchDelete)) *BatchDelete {
 	s3client := s3.NewFromConfig(*c)
-	return NewBatchDeleteWithClient(s3client, options...)
+
+	return NewBatchDeleteWithClient(s3client, batchSize, options...)
 }
 
 // BatchDeleteObject is a wrapper object for calling the batch delete operation.

resources/s3-bucket.go

@@ -231,7 +231,8 @@ func (r *S3Bucket) RemoveAllVersions(ctx context.Context) error {
 	}
 
 	iterator := newS3DeleteVersionListIterator(r.svc, params, setBypass)
-	return awsmod.NewBatchDeleteWithClient(r.svc).Delete(ctx, iterator, opts...)
+	batchSize := r.settings.GetInt("S3VersionDeleteBatchSize")
+	return awsmod.NewBatchDeleteWithClient(r.svc, batchSize).Delete(ctx, iterator, opts...)
 }
 
 func (r *S3Bucket) RemoveAllObjects(ctx context.Context) error {
@@ -248,7 +249,8 @@ func (r *S3Bucket) RemoveAllObjects(ctx context.Context) error {
 	}
 
 	iterator := newS3ObjectDeleteListIterator(r.svc, params, setBypass)
-	return awsmod.NewBatchDeleteWithClient(r.svc).Delete(ctx, iterator, opts...)
+	batchSize := r.settings.GetInt("S3ObjectDeleteBatchSize")
+	return awsmod.NewBatchDeleteWithClient(r.svc, batchSize).Delete(ctx, iterator, opts...)
 }
 
 func (r *S3Bucket) Settings(settings *libsettings.Setting) {

resources/s3-bucket_test.go

@@ -104,7 +104,7 @@ func (suite *TestS3BucketSuite) TearDownSuite() {
 	iterator := newS3DeleteVersionListIterator(suite.svc, &s3.ListObjectVersionsInput{
 		Bucket: suite.bucket,
 	}, true)
-	if err := awsmod.NewBatchDeleteWithClient(suite.svc).Delete(context.TODO(), iterator, bypassGovernanceRetention); err != nil {
+	if err := awsmod.NewBatchDeleteWithClient(suite.svc, -1).Delete(context.TODO(), iterator, bypassGovernanceRetention); err != nil {
 		if !strings.Contains(err.Error(), "NoSuchBucket") {
 			suite.T().Fatalf("failed to delete objects, %v", err)
 		}
@@ -113,7 +113,7 @@ func (suite *TestS3BucketSuite) TearDownSuite() {
 	iterator2 := newS3ObjectDeleteListIterator(suite.svc, &s3.ListObjectsV2Input{
 		Bucket: suite.bucket,
 	}, true)
-	if err := awsmod.NewBatchDeleteWithClient(suite.svc).Delete(context.TODO(), iterator2, bypassGovernanceRetention); err != nil {
+	if err := awsmod.NewBatchDeleteWithClient(suite.svc, -1).Delete(context.TODO(), iterator2, bypassGovernanceRetention); err != nil {
 		if !strings.Contains(err.Error(), "NoSuchBucket") {
 			suite.T().Fatalf("failed to delete objects, %v", err)
 		}
@@ -178,10 +178,34 @@ func (suite *TestS3BucketBypassGovernanceSuite) TestS3BucketRemoveWithBypass() {
 	assert.Nil(suite.T(), err)
 }
 
+type TestS3BucketBatchSizeSuite struct {
+	TestS3BucketSuite
+}
+
+func (suite *TestS3BucketBatchSizeSuite) TestS3BucketRemoveWithNonDefaultBatchSize() {
+	// Create the S3Bucket object
+	bucket := &S3Bucket{
+		svc: suite.svc,
+		settings: &libsettings.Setting{
+			"BypassGovernanceRetention": true,
+			"S3ObjectDeleteBatchSize":   1000,
+		},
+		Name:       suite.bucket,
+		ObjectLock: s3types.ObjectLockEnabledEnabled,
+	}
+
+	err := bucket.Remove(context.TODO())
+	assert.Nil(suite.T(), err)
+}
+
 func TestS3BucketObjectLock(t *testing.T) {
 	suite.Run(t, new(TestS3BucketObjectLockSuite))
 }
 
 func TestS3BucketBypassGovernance(t *testing.T) {
 	suite.Run(t, new(TestS3BucketBypassGovernanceSuite))
 }
+
+func TestS3BucketBatchSize(t *testing.T) {
+	suite.Run(t, new(TestS3BucketBatchSizeSuite))
+}