[Storage] [DataMovement] [Blobs] Setting AccessTier to null will disable preserving (Azure#49231)

* [Storage] [DataMovement] [Blobs] Setting AccessTier to null will disable preserving from source to destination

* Cleanup

* Forgot to push assets.json

* Updated checkpoint 4 bin file

* Update tests

* WIP

* Added fix for preserve page blob premium tier

* Removed unnecessary access tier calculation line

Author: amnguye

sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md

@@ -3,10 +3,12 @@
 ## 12.2.0-beta.1 (Unreleased)
 
 ### Features Added
+- Explicitly setting `BlobStorageResourceOptions.AccessTier` to `null` will disable preserving the access tier value from the source blob to the destination blob.
 
 ### Breaking Changes
 
 ### Bugs Fixed
+- Fixed bug where AccessTier for Premium Page Blobs were not being preserved.
 
 ### Other Changes
 

sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_476246f258"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_385f623849"
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj

@@ -21,8 +21,8 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Azure.Core" />
-    <PackageReference Include="Azure.Storage.Blobs" />
-    <!-- <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" /> -->
+    <!-- <PackageReference Include="Azure.Storage.Blobs" />-->
+    <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Blobs\src\Azure.Storage.Blobs.csproj" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.DataMovement\src\Azure.Storage.DataMovement.csproj" />
     <PackageReference Include="System.Threading.Channels" />
   </ItemGroup>

sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointDetails.cs

@@ -48,6 +48,7 @@ internal class BlobDestinationCheckpointDetails : StorageResourceCheckpointDetai
         /// The access tier of the destination blob.
         /// </summary>
         public AccessTier? AccessTierValue;
+        public bool IsAccessTierSet;
 
         /// <summary>
         /// The metadata for the destination blob.
@@ -85,6 +86,7 @@ public BlobDestinationCheckpointDetails(bool isBlobTypeSet, BlobType? blobType,
                 contentDisposition: blobOptions?.ContentDisposition,
                 isCacheControlSet: blobOptions?._isCacheControlSet ?? false,
                 cacheControl: blobOptions?.CacheControl,
+                isAccessTierSet: blobOptions?._isAccessTierSet ?? false,
                 accessTier: blobOptions?.AccessTier,
                 isMetadataSet: blobOptions?._isMetadataSet ?? false,
                 metadata: blobOptions?.Metadata,
@@ -105,6 +107,7 @@ public BlobDestinationCheckpointDetails(
             string contentDisposition,
             bool isCacheControlSet,
             string cacheControl,
+            bool isAccessTierSet,
             AccessTier? accessTier,
             bool isMetadataSet,
             Metadata metadata,
@@ -115,6 +118,7 @@ public BlobDestinationCheckpointDetails(
             BlobType = blobType;
             IsBlobTypeSet = isBlobTypeSet;
 
+            IsAccessTierSet = isAccessTierSet;
             AccessTierValue = accessTier;
 
             CacheControl = cacheControl;
@@ -233,6 +237,7 @@ protected override void Serialize(Stream stream)
             }
 
             // AccessTier
+            writer.Write(IsAccessTierSet);
             writer.Write((byte)AccessTierValue.ToJobPlanAccessTier());
 
             // Preserve Metadata
@@ -335,9 +340,10 @@ internal static BlobDestinationCheckpointDetails Deserialize(Stream stream)
             int cacheControlLength = reader.ReadInt32();
 
             // AccessTier
+            bool isAccessTierSet = reader.ReadBoolean();
             AccessTier? accessTier = default;
             JobPlanAccessTier jobPlanAccessTier = (JobPlanAccessTier)reader.ReadByte();
-            if (!jobPlanAccessTier.Equals(JobPlanAccessTier.None))
+            if (isAccessTierSet)
             {
                 accessTier = new AccessTier(jobPlanAccessTier.ToString());
             }
@@ -422,6 +428,7 @@ internal static BlobDestinationCheckpointDetails Deserialize(Stream stream)
                 contentDisposition: contentDisposition,
                 isCacheControlSet: isCacheControlSet,
                 cacheControl: cacheControl,
+                isAccessTierSet: isAccessTierSet,
                 accessTier: accessTier,
                 isMetadataSet: isMetadataSet,
                 metadata: metadataString.ToDictionary(nameof(metadataString)),

sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceOptions.cs

@@ -30,6 +30,9 @@ public class BlobStorageResourceOptions
         private string _contentType = default;
         internal bool _isContentTypeSet = false;
 
+        private AccessTier? _accessTier = default;
+        internal bool _isAccessTierSet = false;
+
         /// <summary>
         /// Default constructor.
         /// </summary>
@@ -52,6 +55,7 @@ internal BlobStorageResourceOptions(BlobStorageResourceOptions other)
             ContentType = other?.ContentType;
             _isContentTypeSet = other?._isContentTypeSet ?? false;
             AccessTier = other?.AccessTier;
+            _isAccessTierSet = other?._isAccessTierSet ?? false;
         }
 
         internal BlobStorageResourceOptions(BlobDestinationCheckpointDetails checkpointDetails)
@@ -69,6 +73,7 @@ internal BlobStorageResourceOptions(BlobDestinationCheckpointDetails checkpointD
             ContentType = checkpointDetails.ContentType;
             _isContentTypeSet = checkpointDetails.IsContentTypeSet;
             AccessTier = checkpointDetails.AccessTierValue;
+            _isAccessTierSet = checkpointDetails.IsAccessTierSet;
         }
 
         /// <summary>
@@ -189,11 +194,19 @@ public string ContentType
         /// Optional. See <see cref="Storage.Blobs.Models.AccessTier"/>.
         /// Indicates the access tier to be set on the destination blob.
         ///
-        /// Access Tier is automatically preserved during blob to blob copies.
+        /// Access Tier is automatically preserved during blob to blob copies. If explicitly set to null, the Access Tier will not be preserved from source to destination.
         ///
         /// Applies to upload and copy transfers.
         /// Also respective Tier Values applies only to Block or Page Blobs.
         /// </summary>
-        public AccessTier? AccessTier { get; set; }
+        public AccessTier? AccessTier
+        {
+            get => _accessTier;
+            set
+            {
+                _accessTier = value;
+                _isAccessTierSet = true;
+            }
+        }
     }
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobConstants.cs

@@ -21,8 +21,11 @@ internal class SourceCheckpointDetails
 
         internal class DestinationCheckpointDetails
         {
+            // Blob Schema Versions 1 and 2 were the beta version of the schema and do not need to be serialized and deserialized backwards compatible.
+            // Only Blob Schema Versions 3 and beyond need to be backwards compatible.
             internal const int SchemaVersion_3 = 3;
-            internal const int SchemaVersion = SchemaVersion_3;
+            internal const int SchemaVersion_4 = 4;
+            internal const int SchemaVersion = SchemaVersion_4;
 
             internal const int VersionIndex = 0;
             internal const int PreserveBlobTypeIndex = VersionIndex + IntSizeInBytes;
@@ -47,7 +50,8 @@ internal class DestinationCheckpointDetails
             internal const int CacheControlOffsetIndex = PreserveCacheControlIndex + OneByte;
             internal const int CacheControlLengthIndex = CacheControlOffsetIndex + IntSizeInBytes;
 
-            internal const int AccessTierValueIndex = CacheControlLengthIndex + IntSizeInBytes;
+            internal const int PreserveAccessTierIndex = CacheControlLengthIndex + IntSizeInBytes;
+            internal const int AccessTierValueIndex = PreserveAccessTierIndex + OneByte;
 
             internal const int PreserveMetadataIndex = AccessTierValueIndex + OneByte;
             internal const int MetadataOffsetIndex = PreserveMetadataIndex + OneByte;

sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs

@@ -482,6 +482,7 @@ internal static PageBlobCreateOptions GetCreateOptions(
                     TagConditions = options?.DestinationConditions?.TagConditions,
                     LeaseId = options?.DestinationConditions?.LeaseId,
                 },
+                PremiumPageBlobAccessTier = GetAccessTier(options, sourceProperties?.RawProperties).ToPremiumPageBlobAccessTier(),
             };
         }
 
@@ -667,7 +668,7 @@ private static BlobHttpHeaders GetHttpHeaders(
         private static AccessTier? GetAccessTier(
             BlobStorageResourceOptions options,
             IDictionary<string, object> properties)
-            => options?.AccessTier != default
+            => options?._isAccessTierSet ?? false
                 ? options?.AccessTier
                 : properties?.TryGetValue(DataMovementConstants.ResourceProperties.AccessTier, out object accessTierObject) == true
                     ? (AccessTier?)accessTierObject
@@ -682,5 +683,20 @@ private static Metadata GetMetadata(
                 : properties?.TryGetValue(DataMovementConstants.ResourceProperties.Metadata, out object metadataObject) == true
                     ? (Metadata)metadataObject
                     : default;
+
+        // Convert AccessTier to PremiumPageBlobAccessTier
+        // As long as it works. Do not set if the AccessTier is a BlockBlob tier
+        private static PremiumPageBlobAccessTier? ToPremiumPageBlobAccessTier(this AccessTier? accessTier)
+        {
+            if (accessTier != default &&
+                accessTier != AccessTier.Hot &&
+                accessTier != AccessTier.Cool &&
+                accessTier != AccessTier.Archive &&
+                accessTier != AccessTier.Cold)
+            {
+                return new PremiumPageBlobAccessTier(accessTier.ToString());
+            }
+            return default;
+        }
     }
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDestinationCheckpointDetailsTests.cs

@@ -46,6 +46,7 @@ private BlobDestinationCheckpointDetails CreatePreserveValues()
                 default,
                 false,
                 default,
+                false,
                 default,
                 false,
                 default,
@@ -66,6 +67,7 @@ private BlobDestinationCheckpointDetails CreateSetSampleValues()
                 contentDisposition: DefaultContentDisposition,
                 isCacheControlSet: true,
                 cacheControl: DefaultCacheControl,
+                isAccessTierSet: true,
                 accessTier: DefaultAccessTier,
                 isMetadataSet: true,
                 metadata: DefaultMetadata,
@@ -86,6 +88,7 @@ private BlobDestinationCheckpointDetails CreateSetDefaultValues()
                 default,
                 true,
                 default,
+                true,
                 default,
                 true,
                 default,
@@ -94,7 +97,7 @@ private BlobDestinationCheckpointDetails CreateSetDefaultValues()
 
         private void TestAssertSerializedData(BlobDestinationCheckpointDetails data)
         {
-            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.3.bin");
+            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.4.bin");
             using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointDetails.VariableLengthStartIndex))
             using (FileStream fileStream = File.OpenRead(samplePath))
             {
@@ -126,6 +129,7 @@ public void Ctor()
             Assert.IsEmpty(data.ContentDispositionBytes);
             Assert.AreEqual(false, data.IsCacheControlSet);
             Assert.IsEmpty(data.CacheControlBytes);
+            Assert.AreEqual(false, data.IsAccessTierSet);
             Assert.IsNull(data.AccessTierValue);
             Assert.AreEqual(false, data.IsMetadataSet);
             Assert.IsNull(data.Metadata);
@@ -138,7 +142,7 @@ public void Ctor_SetValues()
         {
             BlobDestinationCheckpointDetails data = CreateSetSampleValues();
 
-            VerifySampleValues(data, DataMovementBlobConstants.DestinationCheckpointDetails.SchemaVersion);
+            VerifySampleValues_Version4(data);
         }
 
         [Test]
@@ -156,7 +160,7 @@ public void Serialize_NoPreserveTags()
             data.PreserveTags = true;
             data.TagsBytes = default;
 
-            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.3.bin");
+            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.4.bin");
             using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointDetails.VariableLengthStartIndex))
             using (FileStream fileStream = File.OpenRead(samplePath))
             {
@@ -195,7 +199,7 @@ public void Serialize_SetAccessTier()
             BlobDestinationCheckpointDetails data = CreateSetSampleValues();
             data.AccessTierValue = AccessTier.Cold;
 
-            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.3.bin");
+            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.4.bin");
             using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointDetails.VariableLengthStartIndex))
             using (FileStream fileStream = File.OpenRead(samplePath))
             {
@@ -225,10 +229,11 @@ public void Deserialize()
                 data.Serialize(stream);
                 stream.Position = 0;
                 BlobDestinationCheckpointDetails deserialized = BlobDestinationCheckpointDetails.Deserialize(stream);
-                VerifySampleValues(deserialized, DataMovementBlobConstants.DestinationCheckpointDetails.SchemaVersion);
+                VerifySampleValues_Version4(deserialized);
             }
         }
 
+        [Ignore("Renable after implementing backwards compatibility for older versions")]
         [Test]
         public void Deserialize_File_Version_3()
         {
@@ -237,13 +242,48 @@ public void Deserialize_File_Version_3()
             {
                 stream.Position = 0;
                 BlobDestinationCheckpointDetails deserialized = BlobDestinationCheckpointDetails.Deserialize(stream);
-                VerifySampleValues(deserialized, 3);
+                VerifySampleValues_Version3(deserialized);
             }
         }
 
-        private void VerifySampleValues(BlobDestinationCheckpointDetails data, int version)
+        [Test]
+        public void Deserialize_File_Version_4()
+        {
+            string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointDetails.4.bin");
+            using (FileStream stream = File.OpenRead(samplePath))
+            {
+                stream.Position = 0;
+                BlobDestinationCheckpointDetails deserialized = BlobDestinationCheckpointDetails.Deserialize(stream);
+                VerifySampleValues_Version4(deserialized);
+            }
+        }
+
+        private void VerifySampleValues_Version3(BlobDestinationCheckpointDetails data)
+        {
+            Assert.AreEqual(3, data.Version);
+            Assert.IsTrue(data.IsBlobTypeSet);
+            Assert.AreEqual(DefaultBlobType, data.BlobType);
+            Assert.AreEqual(true, data.IsContentTypeSet);
+            Assert.AreEqual(StringToByteArray(DefaultContentType), data.ContentTypeBytes);
+            Assert.AreEqual(true, data.IsContentEncodingSet);
+            Assert.AreEqual(StringToByteArray(DefaultContentEncoding), data.ContentEncodingBytes);
+            Assert.AreEqual(true, data.IsContentLanguageSet);
+            Assert.AreEqual(StringToByteArray(DefaultContentLanguage), data.ContentLanguageBytes);
+            Assert.AreEqual(true, data.IsContentDispositionSet);
+            Assert.AreEqual(StringToByteArray(DefaultContentDisposition), data.ContentDispositionBytes);
+            Assert.AreEqual(true, data.IsCacheControlSet);
+            Assert.AreEqual(StringToByteArray(DefaultCacheControl), data.CacheControlBytes);
+            Assert.AreEqual(true, data.IsAccessTierSet);
+            Assert.AreEqual(DefaultAccessTier, data.AccessTierValue);
+            Assert.AreEqual(true, data.IsMetadataSet);
+            CollectionAssert.AreEquivalent(DefaultMetadata, data.Metadata);
+            Assert.AreEqual(false, data.PreserveTags);
+            CollectionAssert.AreEquivalent(DefaultTags, data.Tags);
+        }
+
+        private void VerifySampleValues_Version4(BlobDestinationCheckpointDetails data)
         {
-            Assert.AreEqual(version, data.Version);
+            Assert.AreEqual(4, data.Version);
             Assert.IsTrue(data.IsBlobTypeSet);
             Assert.AreEqual(DefaultBlobType, data.BlobType);
             Assert.AreEqual(true, data.IsContentTypeSet);
@@ -256,6 +296,7 @@ private void VerifySampleValues(BlobDestinationCheckpointDetails data, int versi
             Assert.AreEqual(StringToByteArray(DefaultContentDisposition), data.ContentDispositionBytes);
             Assert.AreEqual(true, data.IsCacheControlSet);
             Assert.AreEqual(StringToByteArray(DefaultCacheControl), data.CacheControlBytes);
+            Assert.AreEqual(true, data.IsAccessTierSet);
             Assert.AreEqual(DefaultAccessTier, data.AccessTierValue);
             Assert.AreEqual(true, data.IsMetadataSet);
             CollectionAssert.AreEquivalent(DefaultMetadata, data.Metadata);

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobStartTransferCopyTests.cs

@@ -172,6 +172,7 @@ protected override StorageResourceItem GetDestinationStorageResourceItem(
             {
                 options = new BlockBlobStorageResourceOptions
                 {
+                    AccessTier = default,
                     ContentDisposition = default,
                     ContentLanguage = default,
                     CacheControl = default,
@@ -211,7 +212,9 @@ protected override async Task VerifyPropertiesCopyAsync(
                 Assert.IsNull(destinationProperties.ContentDisposition);
                 Assert.IsNull(destinationProperties.ContentLanguage);
                 Assert.IsNull(destinationProperties.CacheControl);
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
+                // Because AccessTier is not preserved, the access tier value on the destination will
+                // default to what the storage account sets (normally AccessTier.Hot)
+                Assert.AreNotEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.That(destinationProperties.ContentType, Is.Not.EqualTo(_defaultContentType));
 
                 GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();