From 0bb3eaaf2a5b9541124d2397d9059bf578f6c747 Mon Sep 17 00:00:00 2001 From: vincenttran-msft Date: Thu, 13 Feb 2025 19:46:08 -0800 Subject: [PATCH 1/4] Repro --- .../tests/test_common_blob_async.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py index 4ae969580409..b8b1cecdb2e8 100644 --- a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py @@ -3455,4 +3455,59 @@ async def test_upload_blob_partial_stream_chunked(self, **kwargs): # Assert result = await (await blob.download_blob()).readall() assert result == data[:length] + + @pytest.mark.live_test_only + @BlobPreparer() + async def test_download_blob_decompress(self, **kwargs): + # This test will currently fail against the current codebase + # AssertionError: assert b'hello from gzip' == + # b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00' + # Becuase on decompress=False, data.response.read() does not respect turning off decompression. So result (LHS) comes back decompressed. + + # Proposed fix: + # If we change how we access the content to: content = b"".join([d async for d in data]) in _download_async + # from await data.response.read() / content = cast(bytes, data.response.content) + # This will respect the decompress= + + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + # Arrange + await self._setup(storage_account_name, storage_account_key) + blob_name = self._get_blob_reference() + blob = self.bsc.get_blob_client(self.container_name, blob_name) + compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00' + decompressed_data = b"hello from gzip" + content_settings = ContentSettings(content_encoding='gzip') + + # Act / Assert + await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True) + + downloaded = await blob.download_blob(decompress=True) + result = await downloaded.readall() + assert result == decompressed_data + + downloaded = await blob.download_blob(decompress=False) + result = await downloaded.readall() + assert result == compressed_data + + @pytest.mark.live_test_only + @BlobPreparer() + async def test_download_blob_decompress_md5(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + # Arrange + await self._setup(storage_account_name, storage_account_key) + blob_name = self._get_blob_reference() + blob = self.bsc.get_blob_client(self.container_name, blob_name) + compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00' + decompressed_data = b"hello from gzip" + content_settings = ContentSettings(content_encoding='gzip') + + # Act / Assert + await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True) + downloaded = await blob.download_blob(validate_content=True) + result = await downloaded.readall() + assert result == decompressed_data # ------------------------------------------------------------------------------ \ No newline at end of file From 37efb36b2e63e0848c19dbf827693a3dd5bebd8a Mon Sep 17 00:00:00 2001 From: vincenttran-msft Date: Fri, 14 Feb 2025 14:55:57 -0800 Subject: [PATCH 2/4] Add test case --- .../tests/test_common_blob_async.py | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py index 282f5fc2ef39..eb1a9f254bfe 100644 --- a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py @@ -3456,6 +3456,61 @@ async def test_upload_blob_partial_stream_chunked(self, **kwargs): result = await (await blob.download_blob()).readall() assert result == data[:length] + @BlobPreparer() + async def test_mock_transport_no_content_validation(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + transport = MockStorageTransport() + blob_client = BlobClient( + self.account_url(storage_account_name, "blob"), + container_name='test_cont', + blob_name='test_blob', + credential=storage_account_key, + transport=transport, + retry_total=0 + ) + + content = await blob_client.download_blob() + assert content is not None + + props = await blob_client.get_blob_properties() + assert props is not None + + data = b"Hello Async World!" + stream = AsyncStream(data) + resp = await blob_client.upload_blob(stream, overwrite=True) + assert resp is not None + + blob_data = await (await blob_client.download_blob()).read() + assert blob_data == b"Hello Async World!" # data is fixed by mock transport + + resp = await blob_client.delete_blob() + assert resp is None + + @BlobPreparer() + async def test_mock_transport_with_content_validation(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + transport = MockStorageTransport() + blob_client = BlobClient( + self.account_url(storage_account_name, "blob"), + container_name='test_cont', + blob_name='test_blob', + credential=storage_account_key, + transport=transport, + retry_total=0 + ) + + data = b"Hello Async World!" + stream = AsyncStream(data) + resp = await blob_client.upload_blob(stream, overwrite=True, validate_content=True) + assert resp is not None + + blob_data = await (await blob_client.download_blob(validate_content=True)).read() + assert blob_data == b"Hello Async World!" # data is fixed by mock transport + @pytest.mark.live_test_only @BlobPreparer() async def test_download_blob_decompress(self, **kwargs): @@ -3500,4 +3555,4 @@ async def test_download_blob_decompress_md5(self, **kwargs): downloaded = await blob.download_blob(validate_content=True) result = await downloaded.readall() assert result == decompressed_data -# ------------------------------------------------------------------------------ +# ------------------------------------------------------------------------------ \ No newline at end of file From f57f65c2e8d2c193b73bad17329c8f60aeb49c09 Mon Sep 17 00:00:00 2001 From: vincenttran-msft Date: Fri, 14 Feb 2025 15:44:45 -0800 Subject: [PATCH 3/4] Pass decompress=False despite no side-effect --- .../azure-storage-blob/tests/test_common_blob_async.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py index eb1a9f254bfe..645e3c20bd4d 100644 --- a/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_common_blob_async.py @@ -64,7 +64,7 @@ class TestStorageCommonBlobAsync(AsyncStorageRecordedTestCase): # --Helpers----------------------------------------------------------------- async def _setup(self, storage_account_name, key): - self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key) + self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key, retry_total=0) self.container_name = self.get_resource_name('utcontainer') self.source_container_name = self.get_resource_name('utcontainersource') self.byte_data = self.get_random_bytes(1024) @@ -3552,7 +3552,7 @@ async def test_download_blob_decompress_md5(self, **kwargs): # Act / Assert await blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True) - downloaded = await blob.download_blob(validate_content=True) + downloaded = await blob.download_blob(validate_content=True, decompress=False) result = await downloaded.readall() assert result == decompressed_data # ------------------------------------------------------------------------------ \ No newline at end of file From 3bcd40100d65c205edcd9ed865f2c30a5983282c Mon Sep 17 00:00:00 2001 From: vincenttran-msft Date: Fri, 14 Feb 2025 16:10:23 -0800 Subject: [PATCH 4/4] Failing testcase in sync --- .../tests/test_common_blob.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_common_blob.py b/sdk/storage/azure-storage-blob/tests/test_common_blob.py index b7e8936b3656..3ed57da65e05 100644 --- a/sdk/storage/azure-storage-blob/tests/test_common_blob.py +++ b/sdk/storage/azure-storage-blob/tests/test_common_blob.py @@ -62,7 +62,7 @@ class TestStorageCommonBlob(StorageRecordedTestCase): def _setup(self, storage_account_name, key): - self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key) + self.bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=key, retry_total=0) self.container_name = self.get_resource_name('utcontainer') self.source_container_name = self.get_resource_name('utcontainersource') if self.is_live: @@ -3531,4 +3531,24 @@ def test_upload_blob_partial_stream_chunked(self, **kwargs): result = blob.download_blob().readall() assert result == data[:length] + @pytest.mark.live_test_only + @BlobPreparer() + def test_download_blob_decompress(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + # Arrange + self._setup(storage_account_name, storage_account_key) + blob_name = self._get_blob_reference() + blob = self.bsc.get_blob_client(self.container_name, blob_name) + compressed_data = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00' + decompressed_data = b"hello from gzip" + content_settings = ContentSettings(content_encoding='gzip') + + # Act / Assert + blob.upload_blob(data=compressed_data, content_settings=content_settings, overwrite=True) + + result = blob.download_blob(validate_content =True, decompress=False).readall() + assert result == compressed_data + # ------------------------------------------------------------------------------ \ No newline at end of file