Merge branch 'trunk' into fix/s3-stream-chunk-size

Author: micafer

CHANGES.rst

@@ -42,6 +42,15 @@ Storage
   requested chunk size is now passed through to the underlying iterator
   instead of the hardcoded default.
   (GITHUB-1798)
+
+  [Sanjay Santhanam - @Sanjays2402]
+
+- [Azure Blobs] Fix ``chunk_size`` argument being ignored by
+  ``download_object_as_stream`` and ``download_object_range_as_stream``. The
+  requested chunk size is now forwarded to the underlying iterator instead of
+  always using ``AZURE_DOWNLOAD_CHUNK_SIZE``.
+  (GITHUB-1698)
+
   [Sanjay Santhanam - @Sanjays2402]
 
 Changes in Apache Libcloud 3.9.1

libcloud/storage/drivers/azure_blobs.py

@@ -793,7 +793,7 @@ def download_object_as_stream(self, obj, chunk_size=None):
         """
         obj_path = self._get_object_path(obj.container, obj.name)
         response = self.connection.request(obj_path, method="GET", stream=True, raw=True)
-        iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE)
+        iterator = response.iter_content(chunk_size or AZURE_DOWNLOAD_CHUNK_SIZE)
 
         return self._get_object(
             obj=obj,
@@ -848,7 +848,7 @@ def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None, chun
         response = self.connection.request(
             obj_path, method="GET", headers=headers, stream=True, raw=True
         )
-        iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE)
+        iterator = response.iter_content(chunk_size or AZURE_DOWNLOAD_CHUNK_SIZE)
         success_status_codes = [httplib.OK, httplib.PARTIAL_CONTENT]
 
         return self._get_object(

libcloud/test/storage/test_azure_blobs.py

@@ -697,6 +697,40 @@ def test_download_object_as_stream_success(self):
         consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream)
         self.assertEqual(len(consumed_stream), obj.size)
 
+    def test_download_object_as_stream_uses_chunk_size(self):
+        container = Container(name="foo_bar_container", extra={}, driver=self.driver)
+
+        obj = Object(
+            name="foo_bar_object",
+            size=1000,
+            hash=None,
+            extra={},
+            container=container,
+            meta_data=None,
+            driver=self.driver_type,
+        )
+
+        used_chunk_sizes = []
+
+        def mock_get_object(
+            self, obj, callback, callback_kwargs, response, success_status_code=None
+        ):
+            iterator = callback_kwargs["iterator"]
+            used_chunk_sizes.append(iterator.gi_frame.f_locals.get("chunk_size"))
+            return iterator
+
+        old_func = self.driver_type._get_object
+        self.driver_type._get_object = mock_get_object
+        try:
+            self.driver.download_object_as_stream(obj=obj, chunk_size=1234)
+            self.driver.download_object_range_as_stream(
+                obj=obj, start_bytes=0, end_bytes=10, chunk_size=4321
+            )
+        finally:
+            self.driver_type._get_object = old_func
+
+        self.assertEqual(used_chunk_sizes, [1234, 4321])
+
     def test_download_object_range_success(self):
         container = Container(name="foo_bar_container", extra={}, driver=self.driver)
         obj = Object(