Legal hold & refactored tests (#25)

* Added methods for StorageController

* Added Length to blob metadata & some fixes

* SetLegalHold

* HasLegalHold

* Refactored tests

* Upgrade version

Author: dtymakhov

.github/workflows/dotnet.yml

@@ -70,12 +70,12 @@ jobs:
         AWS_ACCESS_KEY_ID: localkey
         AWS_SECRET_ACCESS_KEY: localsecret
 
-    - name: test-reports
-      uses: dorny/test-reporter@v1.5.0
-      with:
-        name: Test Reporter
-        reporter: dotnet-trx
-        path: ManagedCode.Storage.Tests/test-results.trx
+#    - name: test-reports
+#      uses: dorny/test-reporter@v1.5.0
+#      with:
+#        name: Test Reporter
+#        reporter: dotnet-trx
+#        path: ManagedCode.Storage.Tests/test-results.trx
 
     - name : coverlet
       uses: b3b00/coverlet-action@1.1.9

Directory.Build.props

@@ -13,8 +13,8 @@
         <PackageLicenseExpression>MIT</PackageLicenseExpression>
         <PackageReadmeFile>README.md</PackageReadmeFile>
         <Product>Managed Code - Storage</Product>
-        <Version>1.1.1</Version>
-        <PackageVersion>1.1.1</PackageVersion>
+        <Version>1.1.2</Version>
+        <PackageVersion>1.1.2</PackageVersion>
     </PropertyGroup>
     <PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">
         <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>

ManagedCode.Storage.Aws/AWSStorage.cs

@@ -167,7 +167,8 @@ public async IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs,
             {
                 Name = blobName,
                 Uri = new Uri($"https://s3.amazonaws.com/{_bucket}/{blobName}"),
-                ContentType = objectMetaResponse.Headers.ContentType
+                ContentType = objectMetaResponse.Headers.ContentType,
+                Length = objectMetaResponse.Headers.ContentLength
             };
         }
         catch (AmazonS3Exception ex) when (ex.StatusCode is HttpStatusCode.NotFound)
@@ -204,7 +205,8 @@ public async IAsyncEnumerable<BlobMetadata> GetBlobListAsync(
                 {
                     Name = entry.Key,
                     Uri = new Uri($"https://s3.amazonaws.com/{_bucket}/{entry.Key}"),
-                    ContentType = objectMetaResponse.Headers.ContentType
+                    ContentType = objectMetaResponse.Headers.ContentType,
+                    Length = objectMetaResponse.Headers.ContentLength
                 };
             }
 
@@ -330,4 +332,36 @@ public async Task CreateContainerAsync(CancellationToken cancellationToken = def
     }
 
     #endregion
+
+    public async Task SetLegalHold(string blobName, bool hasLegalHold, CancellationToken cancellationToken = default)
+    {
+        var status = hasLegalHold
+            ? ObjectLockLegalHoldStatus.On
+            : ObjectLockLegalHoldStatus.Off;
+
+        PutObjectLegalHoldRequest request = new()
+        {
+            BucketName = _bucket,
+            Key = blobName,
+            LegalHold = new ObjectLockLegalHold
+            {
+                Status = status,
+            },
+        };
+
+        await _s3Client.PutObjectLegalHoldAsync(request, cancellationToken);
+    }
+
+    public async Task<bool> HasLegalHold(string blobName, CancellationToken cancellationToken = default)
+    {
+        GetObjectLegalHoldRequest request = new()
+        {
+            BucketName = _bucket,
+            Key = blobName
+        };
+
+        var response = await _s3Client.GetObjectLegalHoldAsync(request, cancellationToken);
+
+        return response.LegalHold.Status == ObjectLockLegalHoldStatus.On;
+    }
 }

ManagedCode.Storage.Azure/AzureStorage.cs

@@ -155,17 +155,20 @@ public async IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs,
         await Task.Yield();
 
         var blobClient = _blobContainerClient.GetBlobClient(blobName);
-        
+
         if (!await blobClient.ExistsAsync(cancellationToken))
         {
             return null;
         }
 
+        var properties = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken);
+
         return new BlobMetadata
         {
             Name = blobClient.Name,
             Uri = blobClient.Uri,
             Container = blobClient.BlobContainerName,
+            Length = properties.Value.ContentLength
         };
     }
 
@@ -191,10 +194,13 @@ public async IAsyncEnumerable<BlobMetadata> GetBlobListAsync([EnumeratorCancella
         {
             foreach (var blobItem in item.Values)
             {
-                yield return new BlobMetadata
+                var blobMetadata = new BlobMetadata
                 {
                     Name = blobItem.Name,
+                    Length = blobItem.Properties.ContentLength ?? 0
                 };
+
+                yield return blobMetadata;
             }
         }
     }
@@ -333,4 +339,19 @@ public async Task CreateContainerAsync(CancellationToken cancellationToken = def
     }
 
     #endregion
+
+    public async Task SetLegalHold(string blobName, bool hasLegalHold, CancellationToken cancellationToken = default)
+    {
+        var blobClient = _blobContainerClient.GetBlobClient(blobName);
+
+        await blobClient.SetLegalHoldAsync(hasLegalHold, cancellationToken);
+    }
+
+    public async Task<bool> HasLegalHold(string blobName, CancellationToken cancellationToken = default)
+    {
+        var blobClient = _blobContainerClient.GetBlobClient(blobName);
+        var properties = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken);
+
+        return properties.Value?.HasLegalHold ?? false;
+    }
 }

ManagedCode.Storage.Core/IStorage.cs

@@ -39,4 +39,7 @@ public interface IStorage : IDisposable
     IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs, CancellationToken cancellationToken = default);
 
     Task CreateContainerAsync(CancellationToken cancellationToken = default);
+
+    Task SetLegalHold(string blobName, bool hasLegalHold, CancellationToken cancellationToken = default);
+    Task<bool> HasLegalHold(string blobName, CancellationToken cancellationToken = default);
 }

ManagedCode.Storage.Core/Models/BlobMetadata.cs

@@ -9,4 +9,5 @@ public class BlobMetadata
     public string? Container { get; set; }
     public string? ContentType { get; set; }
     public bool Rewrite { get; set; }
+    public long Length { get; set; }
 }

ManagedCode.Storage.FileSystem/FileSystemStorage.cs

@@ -14,6 +14,7 @@ namespace ManagedCode.Storage.FileSystem;
 public class FileSystemStorage : IFileSystemStorage
 {
     private readonly string _path;
+    private readonly Dictionary<string, FileStream> _lockedFiles = new();
 
     public FileSystemStorage(FileSystemStorageOptions fileSystemStorageOptions)
     {
@@ -157,7 +158,8 @@ public async IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs,
             {
                 Name = fileInfo.Name,
                 Uri = new Uri(Path.Combine(_path, blobName)),
-                ContentType = MimeHelper.GetMimeType(fileInfo.Extension)
+                ContentType = MimeHelper.GetMimeType(fileInfo.Extension),
+                Length = fileInfo.Length
             };
 
             return Task.FromResult<BlobMetadata?>(result);
@@ -280,4 +282,36 @@ public async Task CreateContainerAsync(CancellationToken cancellationToken = def
     }
 
     #endregion
+
+    public async Task SetLegalHold(string blobName, bool hasLegalHold, CancellationToken cancellationToken = default)
+    {
+        if (hasLegalHold && !_lockedFiles.ContainsKey(blobName))
+        {
+            var file = await DownloadAsync(blobName, cancellationToken);
+
+            if (file is null) return;
+
+            var fileStream = File.OpenRead(file.FilePath); // Opening with FileAccess.Read only
+            fileStream.Lock(0, fileStream.Length); // Attempting to lock a region of the read-only file
+
+            _lockedFiles.Add(blobName, fileStream);
+
+            return;
+        }
+
+        if (!hasLegalHold)
+        {
+            if (_lockedFiles.ContainsKey(blobName))
+            {
+                _lockedFiles[blobName].Unlock(0, _lockedFiles[blobName].Length);
+                _lockedFiles[blobName].Dispose();
+                _lockedFiles.Remove(blobName);
+            }
+        }
+    }
+
+    public Task<bool> HasLegalHold(string blobName, CancellationToken cancellationToken = default)
+    {
+        return Task.FromResult(_lockedFiles.ContainsKey(blobName));
+    }
 }

ManagedCode.Storage.Gcp/GCPStorage.cs

@@ -171,7 +171,10 @@ public async IAsyncEnumerable<bool> ExistsAsync(IEnumerable<BlobMetadata> blobs,
             return new BlobMetadata
             {
                 Name = obj.Name,
-                Uri = string.IsNullOrEmpty(obj.MediaLink) ? null : new Uri(obj.MediaLink)
+                Uri = string.IsNullOrEmpty(obj.MediaLink) ? null : new Uri(obj.MediaLink),
+                Container = obj.Bucket,
+                ContentType = obj.ContentType,
+                Length = (long) (obj.Size ?? 0),
             };
         }
         catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.NotFound)
@@ -188,7 +191,10 @@ public IAsyncEnumerable<BlobMetadata> GetBlobListAsync(CancellationToken cancell
                 x => new BlobMetadata
                 {
                     Name = x.Name,
-                    Uri = string.IsNullOrEmpty(x.MediaLink) ? null : new Uri(x.MediaLink)
+                    Uri = string.IsNullOrEmpty(x.MediaLink) ? null : new Uri(x.MediaLink),
+                    Container = x.Bucket,
+                    ContentType = x.ContentType,
+                    Length = (long) (x.Size ?? 0),
                 }
             );
     }
@@ -304,4 +310,19 @@ await _storageClient.CreateBucketAsync(_gcpStorageOptions.BucketOptions.ProjectI
     }
 
     #endregion
+
+    public async Task SetLegalHold(string blobName, bool hasLegalHold, CancellationToken cancellationToken = default)
+    {
+        var storageObject = await _storageClient.GetObjectAsync(_bucket, blobName, cancellationToken: cancellationToken);
+        storageObject.TemporaryHold = hasLegalHold;
+
+        await _storageClient.UpdateObjectAsync(storageObject, cancellationToken: cancellationToken);
+    }
+
+    public async Task<bool> HasLegalHold(string blobName, CancellationToken cancellationToken = default)
+    {
+        var storageObject = await _storageClient.GetObjectAsync(_bucket, blobName, cancellationToken: cancellationToken);
+
+        return storageObject.TemporaryHold ?? false;
+    }
 }

ManagedCode.Storage.Tests/FileHelper.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using ManagedCode.Storage.Core;
 using ManagedCode.Storage.Core.Helpers;
 using Microsoft.AspNetCore.Http;
@@ -9,6 +10,8 @@ namespace ManagedCode.Storage.Tests;
 
 public static class FileHelper
 {
+    private static readonly Random Random = new();
+
     public static LocalFile GenerateLocalFile(string fileName, int byteSize)
     {
         var path = Path.Combine(Path.GetTempPath(), fileName);
@@ -44,4 +47,12 @@ public static string GenerateRandomFileName(string extension = ".txt")
     {
         return $"{Guid.NewGuid().ToString("N").ToLowerInvariant()}.{extension}";
     }
+
+    public static string GenerateRandomFileContent(string extension = ".txt")
+    {
+        const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+
+        return new string(Enumerable.Repeat(chars, 100)
+            .Select(s => s[Random.Next(s.Length)]).ToArray());
+    }
 }

ManagedCode.Storage.Tests/GCP/GoogleStorageTests.cs

@@ -120,7 +120,7 @@ public void StorageAsDefaultTest()
     [Fact]
     public override async Task GetBlobsAsync()
     {
-        var fileList = await CreateFileList(nameof(GetBlobsAsync), 5);
+        var fileList = await CreateFileList(5);
 
         var blobList = fileList.Select(f => f.FileName).ToList();