Skip to content

Commit 751d665

Browse files
GitiiGitii
authored
Allow setting the acl for uploaded files (s3) (#204)
* Allow setting the acl for uploaded files (s3) * rename option and set the acl when creating a bucket. * add test * (Not) Handle unknown pre-defined urls * cleanup
1 parent 610cb10 commit 751d665

File tree

7 files changed

+94
-24
lines changed

7 files changed

+94
-24
lines changed

‎doc/client-settings.md‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,18 +64,19 @@ More options can be found in the [Azure.Identity README](https://github.com/Azur
6464

6565
## Amazon s3 specific properties
6666

67-
| Property | Description |
68-
| --- | ------ |
69-
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
70-
| accessKeyId | Access key id *[Cannot be used with profileName]* |
71-
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
72-
| bucketName | S3 bucket name *[Required]* |
73-
| region | S3 region *[Cannot be used with serviceURL]* |
74-
| serviceURL | S3 service URL *[Cannot be used with region]* |
75-
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
76-
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
77-
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
78-
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
67+
| Property | Description |
68+
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
69+
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
70+
| accessKeyId | Access key id *[Cannot be used with profileName]* |
71+
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
72+
| bucketName | S3 bucket name *[Required]* |
73+
| region | S3 region *[Cannot be used with serviceURL]* |
74+
| serviceURL | S3 service URL *[Cannot be used with region]* |
75+
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
76+
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
77+
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
78+
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
79+
| acl | A acl can be set for uploaded files. By default, no specific canned acl is set and bucket defaults and/or policies are in effect. If the bucket is created by sleet and an acl is set, then the default bucket acl will be set to that acl. |
7980

8081
Either `region` or `serviceURL` should be specified but not both.
8182

‎src/SleetLib/FileSystem/AmazonS3File.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ public class AmazonS3File : FileBase
1616
private readonly string key;
1717
private readonly bool compress = true;
1818
private readonly ServerSideEncryptionMethod serverSideEncryptionMethod;
19+
private readonly S3CannedACL acl;
1920

2021
internal AmazonS3File(
2122
AmazonS3FileSystem fileSystem,
@@ -26,14 +27,16 @@ internal AmazonS3File(
2627
string bucketName,
2728
string key,
2829
ServerSideEncryptionMethod serverSideEncryptionMethod,
29-
bool compress = true)
30+
bool compress = true,
31+
S3CannedACL acl = null)
3032
: base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
3133
{
3234
this.client = client;
3335
this.bucketName = bucketName;
3436
this.key = key;
3537
this.compress = compress;
3638
this.serverSideEncryptionMethod = serverSideEncryptionMethod;
39+
this.acl = acl;
3740
}
3841

3942
protected override async Task CopyFromSource(ILogger log, CancellationToken token)
@@ -131,7 +134,7 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
131134
log.LogWarning($"Unknown file type: {absoluteUri}");
132135
}
133136

134-
await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, token)
137+
await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, acl, token)
135138
.ConfigureAwait(false);
136139

137140
writeStream.Dispose();

‎src/SleetLib/FileSystem/AmazonS3FileSystem.cs‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,30 @@ public class AmazonS3FileSystem : FileSystemBase
2020
private readonly IAmazonS3 _client;
2121
private readonly bool _compress;
2222
private readonly ServerSideEncryptionMethod _serverSideEncryptionMethod;
23+
private readonly S3CannedACL _acl;
2324

2425
private bool? _hasBucket;
2526

26-
public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName)
27-
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None)
27+
public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName, string acl)
28+
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None, acl: acl)
2829
{
2930
}
3031

31-
public AmazonS3FileSystem(
32-
LocalCache cache,
32+
public AmazonS3FileSystem(LocalCache cache,
3333
Uri root,
3434
Uri baseUri,
3535
IAmazonS3 client,
3636
string bucketName,
3737
ServerSideEncryptionMethod serverSideEncryptionMethod,
3838
string feedSubPath = null,
39-
bool compress = true)
39+
bool compress = true,
40+
S3CannedACL acl = null)
4041
: base(cache, root, baseUri)
4142
{
4243
_client = client;
4344
_bucketName = bucketName;
4445
_serverSideEncryptionMethod = serverSideEncryptionMethod;
46+
_acl = acl;
4547

4648
if (!string.IsNullOrEmpty(feedSubPath))
4749
{
@@ -117,7 +119,7 @@ public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, Canc
117119
private ISleetFile CreateAmazonS3File(SleetUriPair pair)
118120
{
119121
var key = GetRelativePath(pair.Root);
120-
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress);
122+
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress, _acl);
121123
}
122124

123125
public override string GetRelativePath(Uri uri)
@@ -176,6 +178,12 @@ public override async Task CreateBucket(ILogger log, CancellationToken token)
176178
// Set the public policy to public read-only
177179
await Retry(SetBucketPolicy, log, token);
178180

181+
// Set the default acl of the bucket. Must not conflict with the public access policy.
182+
if (_acl != null)
183+
{
184+
await Retry(SetBucketAcl, log, token);
185+
}
186+
179187
// Get and release the lock to ensure that everything will work for the next operation.
180188
// In the E2E tests there are often failures due to the bucket saying it is not available
181189
// even though the above checks passed. To work around this wait until a file can be
@@ -223,6 +231,18 @@ private Task SetOwnership(ILogger log, CancellationToken token)
223231
return _client.PutBucketOwnershipControlsAsync(ownerReq, token);
224232
}
225233

234+
// Set the default acl of the bucket.
235+
private Task SetBucketAcl(ILogger log, CancellationToken token)
236+
{
237+
var aclReq = new PutACLRequest()
238+
{
239+
BucketName = _bucketName,
240+
CannedACL = _acl
241+
};
242+
243+
return _client.PutACLAsync(aclReq, token);
244+
}
245+
226246
// Remove public access blocks to allow public policies.
227247
private Task SetPublicAccessBlocks(ILogger log, CancellationToken token)
228248
{

‎src/SleetLib/FileSystem/AmazonS3FileSystemAbstraction.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ public static async Task UploadFileAsync(
131131
string contentEncoding,
132132
Stream reader,
133133
ServerSideEncryptionMethod serverSideEncryptionMethod,
134+
S3CannedACL acl,
134135
CancellationToken token)
135136
{
136137
var transferUtility = new TransferUtility(client);
@@ -158,6 +159,11 @@ public static async Task UploadFileAsync(
158159
if (contentEncoding != null)
159160
request.Headers.ContentEncoding = contentEncoding;
160161

162+
if (acl != null)
163+
{
164+
request.CannedACL = acl;
165+
}
166+
161167
using (transferUtility)
162168
await transferUtility.UploadAsync(request, token).ConfigureAwait(false);
163169
}

‎src/SleetLib/FileSystem/FileSystemFactory.cs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
100100
var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL");
101101
var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
102102
var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
103+
var acl = JsonUtility.GetValueCaseInsensitive(sourceEntry, "acl");
103104

104105
if (string.IsNullOrEmpty(bucketName))
105106
{
@@ -120,6 +121,12 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
120121
throw new ArgumentException("Only 'None' or 'AES256' are currently supported for serverSideEncryptionMethod");
121122
}
122123

124+
S3CannedACL resolvedAcl = null;
125+
if (acl != null)
126+
{
127+
resolvedAcl = S3CannedACL.FindValue(acl);
128+
}
129+
123130
// Use the SDK value
124131
var serverSideEncryptionMethodValue = ServerSideEncryptionMethod.None;
125132
if (serverSideEncryptionMethod == "AES256")
@@ -183,7 +190,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
183190
}
184191
// Load credentials from an ECS docker container
185192
// Check if the env var GenericContainerCredentials.RelativeURIEnvVariable exists
186-
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
193+
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
187194
// deprecated and the property is now internal on GenericContainerCredentials
188195
else if (
189196
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")))
@@ -229,7 +236,9 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
229236
bucketName,
230237
serverSideEncryptionMethodValue,
231238
feedSubPath,
232-
compress);
239+
compress,
240+
resolvedAcl
241+
);
233242
}
234243
}
235244
}

‎test/Sleet.AmazonS3.Tests/AmazonS3TestContext.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public class AmazonS3TestContext : IDisposable
1818

1919
private bool cleanupDone = false;
2020

21-
public AmazonS3TestContext()
21+
public AmazonS3TestContext(string acl = null)
2222
{
2323
BucketName = $"sleet-test-{Guid.NewGuid().ToString()}";
2424
LocalCache = new LocalCache();
@@ -35,7 +35,7 @@ public AmazonS3TestContext()
3535
Client = new AmazonS3Client(accessKeyId, secretAccessKey, config);
3636
Uri = AmazonS3Utility.GetBucketPath(BucketName, region);
3737

38-
FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName);
38+
FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName, acl);
3939
Logger = new TestLogger();
4040
}
4141

‎test/Sleet.AmazonS3.Tests/BasicTests.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,37 @@ public async Task GivenAStorageAccountWithNoContainerVerifyPushSucceeds()
102102
}
103103
}
104104

105+
[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
106+
public async Task GivenAStorageAccountWithNoContainerPublicAclVerifyPushSucceeds()
107+
{
108+
using (var packagesFolder = new TestFolder())
109+
using (var testContext = new AmazonS3TestContext(acl: "public-read"))
110+
{
111+
// Skip creation and allow it to be done during push.
112+
testContext.CreateBucketOnInit = false;
113+
114+
await testContext.InitAsync();
115+
116+
var testPackage = new TestNupkg("packageA", "1.0.0");
117+
var zipFile = testPackage.Save(packagesFolder.Root);
118+
119+
var result = await PushCommand.RunAsync(testContext.LocalSettings,
120+
testContext.FileSystem,
121+
new List<string>() { zipFile.FullName },
122+
force: false,
123+
skipExisting: false,
124+
log: testContext.Logger);
125+
126+
result &= await ValidateCommand.RunAsync(testContext.LocalSettings,
127+
testContext.FileSystem,
128+
testContext.Logger);
129+
130+
result.Should().BeTrue();
131+
132+
await testContext.CleanupAsync();
133+
}
134+
}
135+
105136
[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
106137
public async Task GivenAStorageAccountWithNoInitVerifyPushSucceeds()
107138
{

0 commit comments

Comments
 (0)