add disablePayloadSigning option

doc/client-settings.md

@@ -77,6 +77,7 @@ More options can be found in the [Azure.Identity README](https://github.com/Azur
 | serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None`                                                                                                                                                                                                              |
 | compress                   | Compress JSON files with GZIP before uploading. Default is *true*                                                                                                                                                                                                                                                            |
 | acl                        | A acl can be set for uploaded files. By default, no specific canned acl is set and bucket defaults and/or policies are in effect. If the bucket is created by sleet and an acl is set, then the default bucket acl will be set to that acl. |
+|disablePayloadSigning                       | S3 payload signing is a requirement of AWS SigV4, some S3 compatible storage providers do not implement this. Default is `false`
 
 Either `region` or `serviceURL` should be specified but not both.
 

doc/feed-type-s3.md

@@ -128,7 +128,8 @@ To use [AWS environment variables](https://docs.aws.amazon.com/cli/latest/usergu
       "bucketName": "nupkg",
       "serviceURL": "https://storage.yandexcloud.net",
       "accessKeyId": "IAM_ACCESS_KEY_ID",
-      "secretAccessKey": "IAM_SECRET_ACCESS_KEY"
+      "secretAccessKey": "IAM_SECRET_ACCESS_KEY",
+      "disablePayloadSigning": false, // set to true if SigV4 is not supported by provider (such as Cloudflare R2)
     }
   ]
 }

src/SleetLib/FileSystem/AmazonS3File.cs

@@ -17,6 +17,7 @@ public class AmazonS3File : FileBase
         private readonly bool compress = true;
         private readonly ServerSideEncryptionMethod serverSideEncryptionMethod;
         private readonly S3CannedACL acl;
+        private readonly bool disablePayloadSigning;
 
         internal AmazonS3File(
             AmazonS3FileSystem fileSystem,
@@ -28,7 +29,8 @@ internal AmazonS3File(
             string key,
             ServerSideEncryptionMethod serverSideEncryptionMethod,
             bool compress = true,
-            S3CannedACL acl = null)
+            S3CannedACL acl = null,
+            bool disablePayloadSigning = false)
             : base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
         {
             this.client = client;
@@ -37,6 +39,7 @@ internal AmazonS3File(
             this.compress = compress;
             this.serverSideEncryptionMethod = serverSideEncryptionMethod;
             this.acl = acl;
+            this.disablePayloadSigning = disablePayloadSigning;
         }
 
         protected override async Task CopyFromSource(ILogger log, CancellationToken token)
@@ -134,7 +137,7 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
                     log.LogWarning($"Unknown file type: {absoluteUri}");
                 }
 
-                await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, acl, token)
+                await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, acl, disablePayloadSigning, token)
                     .ConfigureAwait(false);
 
                 writeStream.Dispose();

src/SleetLib/FileSystem/AmazonS3FileSystem.cs

@@ -21,11 +21,13 @@ public class AmazonS3FileSystem : FileSystemBase
         private readonly bool _compress;
         private readonly ServerSideEncryptionMethod _serverSideEncryptionMethod;
         private readonly S3CannedACL _acl;
+        private readonly bool _disablePayloadSigning;
 
         private bool? _hasBucket;
 
-        public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName, string acl)
-            : this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None, acl: acl)
+
+        public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName, string acl, bool disablePayloadSigning = false)
+            : this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None, acl: acl, disablePayloadSigning: disablePayloadSigning)
         {
         }
 
@@ -37,13 +39,15 @@ public AmazonS3FileSystem(LocalCache cache,
             ServerSideEncryptionMethod serverSideEncryptionMethod,
             string feedSubPath = null,
             bool compress = true,
-            S3CannedACL acl = null)
+            S3CannedACL acl = null,
+            bool disablePayloadSigning = false)
             : base(cache, root, baseUri)
         {
             _client = client;
             _bucketName = bucketName;
             _serverSideEncryptionMethod = serverSideEncryptionMethod;
             _acl = acl;
+            _disablePayloadSigning = disablePayloadSigning;
 
             if (!string.IsNullOrEmpty(feedSubPath))
             {
@@ -70,7 +74,7 @@ public override async Task<bool> Validate(ILogger log, CancellationToken token)
 
         public override ISleetFileSystemLock CreateLock(ILogger log)
         {
-            return new AmazonS3FileSystemLock(_client, _bucketName, _serverSideEncryptionMethod, log);
+            return new AmazonS3FileSystemLock(_client, _bucketName, _serverSideEncryptionMethod, _disablePayloadSigning, log);
         }
 
         public override ISleetFile Get(Uri path)
@@ -119,7 +123,7 @@ public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, Canc
         private ISleetFile CreateAmazonS3File(SleetUriPair pair)
         {
             var key = GetRelativePath(pair.Root);
-            return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress, _acl);
+            return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress, _acl, _disablePayloadSigning);
         }
 
         public override string GetRelativePath(Uri uri)
@@ -254,7 +258,7 @@ private Task SetPublicAccessBlocks(ILogger log, CancellationToken token)
                     IgnorePublicAcls = false,
                     RestrictPublicBuckets = false,
                     BlockPublicAcls = false,
-                    BlockPublicPolicy = false,
+                    BlockPublicPolicy = false
                 }
             };
 

src/SleetLib/FileSystem/AmazonS3FileSystemAbstraction.cs

@@ -21,13 +21,15 @@ public static Task CreateFileAsync(
             string key,
             string contentBody,
             ServerSideEncryptionMethod serverSideEncryptionMethod,
+            bool disablePayloadSigning,
             CancellationToken token)
         {
             var putObjectRequest = new PutObjectRequest
             {
                 BucketName = bucketName,
                 Key = key,
-                ContentBody = contentBody
+                ContentBody = contentBody,
+                DisablePayloadSigning = disablePayloadSigning
             };
 
             if (serverSideEncryptionMethod != ServerSideEncryptionMethod.None)
@@ -132,6 +134,7 @@ public static async Task UploadFileAsync(
             Stream reader,
             ServerSideEncryptionMethod serverSideEncryptionMethod,
             S3CannedACL acl,
+            bool disablePayloadSigning,
             CancellationToken token)
         {
             var transferUtility = new TransferUtility(client);
@@ -142,7 +145,8 @@ public static async Task UploadFileAsync(
                 InputStream = reader,
                 AutoCloseStream = false,
                 AutoResetStreamPosition = false,
-                Headers = { CacheControl = "no-store" }
+                Headers = { CacheControl = "no-store" },
+                DisablePayloadSigning = disablePayloadSigning
             };
 
             if (serverSideEncryptionMethod != ServerSideEncryptionMethod.None)

src/SleetLib/FileSystem/AmazonS3FileSystemLock.cs

@@ -17,13 +17,15 @@ public class AmazonS3FileSystemLock : FileSystemLockBase
         private readonly string bucketName;
         private readonly IAmazonS3 client;
         private readonly ServerSideEncryptionMethod serverSideEncryptionMethod;
+        private readonly bool disablePayloadSigning;
 
-        public AmazonS3FileSystemLock(IAmazonS3 client, string bucketName, ServerSideEncryptionMethod serverSideEncryptionMethod, ILogger log)
+        public AmazonS3FileSystemLock(IAmazonS3 client, string bucketName, ServerSideEncryptionMethod serverSideEncryptionMethod, bool disablePayloadSigning, ILogger log)
             : base(log)
         {
             this.client = client;
             this.bucketName = bucketName;
             this.serverSideEncryptionMethod = serverSideEncryptionMethod;
+            this.disablePayloadSigning = disablePayloadSigning;
         }
 
         protected override async Task<Tuple<bool, JObject>> TryObtainLockAsync(string lockMessage, CancellationToken token)
@@ -43,7 +45,7 @@ protected override async Task<Tuple<bool, JObject>> TryObtainLockAsync(string lo
                     // Create a new lock
                     json = GetMessageJson(lockMessage);
 
-                    await CreateFileAsync(client, bucketName, LockFile, json.ToString(), serverSideEncryptionMethod, token).ConfigureAwait(false);
+                    await CreateFileAsync(client, bucketName, LockFile, json.ToString(), serverSideEncryptionMethod, disablePayloadSigning, token).ConfigureAwait(false);
 
                     result = true;
                 }

src/SleetLib/FileSystem/FileSystemFactory.cs

@@ -101,6 +101,8 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
                         var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
                         var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
                         var acl = JsonUtility.GetValueCaseInsensitive(sourceEntry, "acl");
+                        var disablePayloadSigning = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "disablePayloadSigning", false);
+
 
                         if (string.IsNullOrEmpty(bucketName))
                         {
@@ -237,7 +239,8 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
                             serverSideEncryptionMethodValue,
                             feedSubPath,
                             compress,
-                            resolvedAcl
+                            resolvedAcl,
+                            disablePayloadSigning
                         );
                     }
                 }