From 9c44ab36bd82307e772ffe3bf469283ec7b7a54d Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 24 Mar 2025 15:58:39 -0400 Subject: [PATCH 01/43] [WIP] _blob_client.py --- .../azure/storage/blob/_blob_client.py | 171 ++++++++++++----- .../storage/blob/_blob_service_client.py | 9 +- .../azure/storage/blob/_container_client.py | 5 +- .../azure/storage/blob/_serialize.py | 5 +- .../storage/blob/aio/_blob_client_async.py | 180 +++++++++++++----- .../blob/aio/_blob_service_client_async.py | 5 +- .../blob/aio/_container_client_async.py | 5 +- ...p.250ab16d-214e-4844-a12c-feecb1af04c1.dat | 0 8 files changed, 282 insertions(+), 98 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 90049ff88e32..6d14ad865884 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -78,8 +78,9 @@ ) if TYPE_CHECKING: + from azure.core import MatchConditions from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential, TokenCredential - from azure.storage.blob import ContainerClient + from azure.storage.blob import ContainerClient, CustomerProvidedEncryptionKey from ._models import ( ContentSettings, ImmutabilityPolicy, @@ -165,12 +166,25 @@ def __init__( blob_name: str, snapshot: Optional[Union[str, Dict[str, Any]]] = None, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, + version_id: Optional[str] = None, + audience: Optional[int] = None, + max_block_size: Optional[int] = None, + max_page_size: Optional[int] = None, + max_chunk_get_size: Optional[int] = None, + max_single_put_size: Optional[int] = None, + max_single_get_size: Optional[int] = None, + min_large_block_upload_threshold: Optional[int] = None, + use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> None: parsed_url, sas_token, path_snapshot = _parse_url( account_url=account_url, container_name=container_name, - blob_name=blob_name) + blob_name=blob_name + ) self.container_name = container_name self.blob_name = blob_name @@ -180,14 +194,28 @@ def __init__( self.snapshot = snapshot['snapshot'] else: self.snapshot = snapshot or path_snapshot - self.version_id = kwargs.pop('version_id', None) + self.version_id = version_id # This parameter is used for the hierarchy traversal. Give precedence to credential. self._raw_credential = credential if credential else sas_token self._query_str, credential = self._format_query_string(sas_token, credential, snapshot=self.snapshot) - super(BlobClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) + super(BlobClient, self).__init__( + parsed_url, + service='blob', + credential=credential, + secondary_hostname=secondary_hostname, + audience=audience, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, + **kwargs + ) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) - self._client._config.version = get_api_version(kwargs) # type: ignore [assignment] + self._client._config.version = get_api_version(api_version) # type: ignore [assignment] self._configure_encryption(kwargs) def _format_url(self, hostname: str) -> str: @@ -204,6 +232,9 @@ def from_blob_url( cls, blob_url: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long snapshot: Optional[Union[str, Dict[str, Any]]] = None, + *, + version_id: Optional[str] = None, + audience: Optional[str] = None, **kwargs: Any ) -> Self: """Create BlobClient from a blob url. This doesn't support customized blob url with '/' in blob name. @@ -226,7 +257,7 @@ def from_blob_url( ~azure.core.credentials.AzureNamedKeyCredential or ~azure.core.credentials.AzureSasCredential or ~azure.core.credentials.TokenCredential or - str or dict[str, str] or None + str or Dict[str, str] or None :param str snapshot: The optional blob snapshot on which to operate. This can be the snapshot ID string or the response returned from :func:`create_snapshot`. If specified, this will override @@ -241,8 +272,14 @@ def from_blob_url( """ account_url, container_name, blob_name, path_snapshot = _from_blob_url(blob_url=blob_url, snapshot=snapshot) return cls( - account_url, container_name=container_name, blob_name=blob_name, - snapshot=path_snapshot, credential=credential, **kwargs + account_url, + container_name=container_name, + blob_name=blob_name, + snapshot=path_snapshot, + credential=credential, + version_id=version_id, + audience=audience, + **kwargs ) @classmethod @@ -252,6 +289,9 @@ def from_connection_string( blob_name: str, snapshot: Optional[Union[str, Dict[str, Any]]] = None, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + *, + version_id: Optional[str] = None, + audience: Optional[str] = None, **kwargs: Any ) -> Self: """Create BlobClient from a Connection String. @@ -278,7 +318,7 @@ def from_connection_string( ~azure.core.credentials.AzureNamedKeyCredential or ~azure.core.credentials.AzureSasCredential or ~azure.core.credentials.TokenCredential or - str or dict[str, str] or None + str or Dict[str, str] or None :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. :keyword str audience: The audience to use when requesting tokens for Azure Active Directory @@ -300,8 +340,14 @@ def from_connection_string( if 'secondary_hostname' not in kwargs: kwargs['secondary_hostname'] = secondary return cls( - account_url, container_name=container_name, blob_name=blob_name, - snapshot=snapshot, credential=credential, **kwargs + account_url, + container_name=container_name, + blob_name=blob_name, + snapshot=snapshot, + credential=credential, + version_id=version_id, + audience=audience, + **kwargs ) @distributed_trace @@ -312,7 +358,7 @@ def get_account_information(self, **kwargs: Any) -> Dict[str, str]: The keys in the returned dictionary include 'sku_name' and 'account_kind'. :returns: A dict of account information (SKU and account type). - :rtype: dict(str, str) + :rtype: Dict[str, str] """ try: return cast(Dict[str, str], self._client.blob.get_account_info(cls=return_response_headers, **kwargs)) @@ -324,6 +370,25 @@ def upload_blob_from_url( self, source_url: str, *, metadata: Optional[Dict[str, str]] = None, + overwrite: Optional[bool] = None, + include_source_blob_properties: Optional[bool] = None, + tags: Optional[Dict[str, str]] = None, + source_content_md5: Optional[bytearray] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + destination_lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + content_settings: Optional["ContentSettings"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + source_authorization: Optional[str] = None, **kwargs: Any ) -> Dict[str, Any]: """ @@ -342,7 +407,7 @@ def upload_blob_from_url( https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot= https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken - :keyword dict(str, str) metadata: + :keyword Dict[str, str] metadata: Name-value pairs associated with the blob as metadata. :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. If True, upload_blob will overwrite the existing data. If set to False, the @@ -355,7 +420,7 @@ def upload_blob_from_url( and tag values must be between 0 and 256 characters. Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9), space (' '), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_) - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword bytearray source_content_md5: Specify the md5 that is used to verify the integrity of the source bytes. :keyword ~datetime.datetime source_if_modified_since: @@ -425,12 +490,32 @@ def upload_blob_from_url( :returns: Blob-updated property Dict (Etag and last modified) :rtype: Dict[str, Any] """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_blob_from_url_options( source_url=source_url, metadata=metadata, - **kwargs) + overwrite=overwrite, + include_source_blob_properties=include_source_blob_properties, + tags=tags, + source_content_md5=source_content_md5, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + destination_lease=destination_lease, + timeout=timeout, + content_settings=content_settings, + cpk=cpk, + encryption_scope=encryption_scope, + standard_blob_tier=standard_blob_tier, + source_authorization=source_authorization, + **kwargs + ) try: return cast(Dict[str, Any], self._client.block_blob.put_blob_from_url(**options)) except HttpResponseError as error: @@ -455,7 +540,7 @@ def upload_blob( should be supplied for optimal performance. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -465,7 +550,7 @@ def upload_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. If True, upload_blob will overwrite the existing data. If set to False, the operation will fail with ResourceExistsError. The exception to the above is with Append @@ -1157,7 +1242,7 @@ def set_blob_metadata( Dict containing name and value pairs. Each call to this operation replaces all existing metadata attached to the blob. To remove all metadata from the blob, call this operation with no metadata headers. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -1317,7 +1402,7 @@ def create_page_blob( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :param ~azure.storage.blob.PremiumPageBlobTier premium_page_blob_tier: A page blob tier value to set the blob to. The tier correlates to the size of the blob and number of allowed IOPS. This is only applicable to page blobs on @@ -1331,7 +1416,7 @@ def create_page_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword int sequence_number: Only for Page blobs. The sequence number is a user-controlled value that you can use to track requests. The value of the sequence number must be between 0 @@ -1389,7 +1474,7 @@ def create_page_blob( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -1421,7 +1506,7 @@ def create_append_blob( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -1431,7 +1516,7 @@ def create_append_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -1485,7 +1570,7 @@ def create_append_blob( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -1517,7 +1602,7 @@ def create_snapshot( :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword ~datetime.datetime if_modified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. @@ -1564,7 +1649,7 @@ def create_snapshot( see `here `__. :returns: Blob-updated property dict (Snapshot ID, Etag, and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] .. admonition:: Example: @@ -1638,7 +1723,7 @@ def start_copy_from_url( source blob or file to the destination blob. If one or more name-value pairs are specified, the destination blob is created with the specified metadata, and metadata is not copied from the source blob or file. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :param bool incremental_copy: Copies the snapshot of the source page blob to a destination page blob. The snapshot is copied such that only the differential changes between @@ -1657,7 +1742,7 @@ def start_copy_from_url( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) or Literal["COPY"] + :paramtype tags: Dict[str, str] or Literal["COPY"] :keyword ~azure.storage.blob.ImmutabilityPolicy immutability_policy: Specifies the immutability policy of a blob, blob snapshot or blob version. @@ -1981,7 +2066,7 @@ def stage_block( see `here `__. :returns: Blob property dict. - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -2047,7 +2132,7 @@ def stage_block_from_url( Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is the prefix of the source_authorization string. :returns: Blob property dict. - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if kwargs.get('cpk') and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") @@ -2124,7 +2209,7 @@ def commit_block_list( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict[str, str] + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -2134,7 +2219,7 @@ def commit_block_list( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -2203,7 +2288,7 @@ def commit_block_list( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -2275,7 +2360,7 @@ def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: Any) -> and tag values must be between 0 and 256 characters. Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9), space (' '), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_) - :type tags: dict(str, str) + :type tags: Dict[str, str] :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to add tags to. @@ -2407,7 +2492,7 @@ def get_page_ranges( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(dict(str, str), list(dict(str, str)) + :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) """ warnings.warn( "get_page_ranges is deprecated, use list_page_ranges instead", @@ -2583,7 +2668,7 @@ def get_page_range_diff_for_managed_disk( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(dict(str, str), list(dict(str, str)) + :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) """ options = _get_page_ranges_options( snapshot=self.snapshot, @@ -2646,7 +2731,7 @@ def set_sequence_number( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ options = _set_sequence_number_options(sequence_number_action, sequence_number=sequence_number, **kwargs) try: @@ -2702,7 +2787,7 @@ def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, datetime see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if kwargs.get('cpk') and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") @@ -2798,7 +2883,7 @@ def upload_page( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -3000,7 +3085,7 @@ def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, Union see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -3097,7 +3182,7 @@ def append_block( see `here `__. :returns: Blob-updated property dict (Etag, last modified, append offset, committed block count). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -3270,7 +3355,7 @@ def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime, int] see `here `__. :returns: Blob-updated property dict (Etag, last modified, append offset, committed block count). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py index f6e17cb756f0..01b35c98bef4 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py @@ -121,6 +121,9 @@ class BlobServiceClient(StorageAccountHostsMixin, StorageEncryptionMixin): def __init__( self, account_url: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + # TODO **kwargs: Any ) -> None: parsed_url, sas_token = _parse_url(account_url=account_url) @@ -128,7 +131,7 @@ def __init__( self._query_str, credential = self._format_query_string(sas_token, credential) super(BlobServiceClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) - self._client._config.version = get_api_version(kwargs) # type: ignore [assignment] + self._client._config.version = get_api_version(api_version) # type: ignore [assignment] self._configure_encryption(kwargs) def _format_url(self, hostname): @@ -776,8 +779,8 @@ def get_blob_client( else: container_name = container _pipeline = Pipeline( - transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access - policies=self._pipeline._impl_policies # pylint: disable = protected-access + transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access + policies=self._pipeline._impl_policies # pylint: disable = protected-access ) return BlobClient( self.url, container_name=container_name, blob_name=blob_name, snapshot=snapshot, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py index 783df6bc753e..89ad69433eed 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py @@ -137,6 +137,9 @@ def __init__( self, account_url: str, container_name: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + # TODO **kwargs: Any ) -> None: parsed_url, sas_token = _parse_url(account_url=account_url, container_name=container_name) @@ -146,7 +149,7 @@ def __init__( self._raw_credential = credential if credential else sas_token self._query_str, credential = self._format_query_string(sas_token, credential) super(ContainerClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) - self._api_version = get_api_version(kwargs) + self._api_version = get_api_version(api_version) self._client = self._build_generated_client() self._configure_encryption(kwargs) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py index 316e321cd8af..e6e9f163aaf0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py @@ -143,18 +143,19 @@ def get_container_cpk_scope_info(kwargs: Dict[str, Any]) -> Optional[ContainerCp return None -def get_api_version(kwargs: Dict[str, Any]) -> str: - api_version = kwargs.get('api_version', None) +def get_api_version(api_version: Optional[str] = None) -> str: if api_version and api_version not in _SUPPORTED_API_VERSIONS: versions = '\n'.join(_SUPPORTED_API_VERSIONS) raise ValueError(f"Unsupported API version '{api_version}'. Please select from:\n{versions}") return api_version or _SUPPORTED_API_VERSIONS[-1] + def get_version_id(self_vid: Optional[str], kwargs: Dict[str, Any]) -> Optional[str]: if 'version_id' in kwargs: return cast(str, kwargs.pop('version_id')) return self_vid + def serialize_blob_tags_header(tags: Optional[Dict[str, str]] = None) -> Optional[str]: if tags is None: return None diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 7cb074487f58..917d0d9a12b7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -156,18 +156,31 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag :caption: Creating the BlobClient from a SAS URL to a blob. """ def __init__( - self, account_url: str, - container_name: str, - blob_name: str, - snapshot: Optional[Union[str, Dict[str, Any]]] = None, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + self, account_url: str, + container_name: str, + blob_name: str, + snapshot: Optional[Union[str, Dict[str, Any]]] = None, + credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, + version_id: Optional[str] = None, + audience: Optional[int] = None, + max_block_size: Optional[int] = None, + max_page_size: Optional[int] = None, + max_chunk_get_size: Optional[int] = None, + max_single_put_size: Optional[int] = None, + max_single_get_size: Optional[int] = None, + min_large_block_upload_threshold: Optional[int] = None, + use_byte_buffer: Optional[bool] = None, + **kwargs: Any ) -> None: kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) parsed_url, sas_token, path_snapshot = _parse_url( account_url=account_url, container_name=container_name, - blob_name=blob_name) + blob_name=blob_name + ) self.container_name = container_name self.blob_name = blob_name @@ -177,14 +190,28 @@ def __init__( self.snapshot = snapshot['snapshot'] else: self.snapshot = snapshot or path_snapshot - self.version_id = kwargs.pop('version_id', None) + self.version_id = version_id # This parameter is used for the hierarchy traversal. Give precedence to credential. self._raw_credential = credential if credential else sas_token self._query_str, credential = self._format_query_string(sas_token, credential, snapshot=self.snapshot) - super(BlobClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) + super(BlobClient, self).__init__( + parsed_url, + service='blob', + credential=credential, + secondary_hostname=secondary_hostname, + audience=audience, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, + **kwargs + ) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) - self._client._config.version = get_api_version(kwargs) # type: ignore [assignment] + self._client._config.version = get_api_version(api_version) # type: ignore [assignment] self._configure_encryption(kwargs) def _format_url(self, hostname: str) -> str: @@ -201,6 +228,9 @@ def from_blob_url( cls, blob_url: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long snapshot: Optional[Union[str, Dict[str, Any]]] = None, + *, + version_id: Optional[str] = None, + audience: Optional[str] = None, **kwargs: Any ) -> Self: """Create BlobClient from a blob url. This doesn't support customized blob url with '/' in blob name. @@ -234,8 +264,14 @@ def from_blob_url( """ account_url, container_name, blob_name, path_snapshot = _from_blob_url(blob_url=blob_url, snapshot=snapshot) return cls( - account_url, container_name=container_name, blob_name=blob_name, - snapshot=path_snapshot, credential=credential, **kwargs + account_url, + container_name=container_name, + blob_name=blob_name, + snapshot=path_snapshot, + credential=credential, + version_id=version_id, + audience=audience, + **kwargs ) @classmethod @@ -245,6 +281,9 @@ def from_connection_string( blob_name: str, snapshot: Optional[Union[str, Dict[str, Any]]] = None, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + *, + version_id: Optional[str] = None, + audience: Optional[str] = None, **kwargs: Any ) -> Self: """Create BlobClient from a Connection String. @@ -289,8 +328,14 @@ def from_connection_string( if 'secondary_hostname' not in kwargs: kwargs['secondary_hostname'] = secondary return cls( - account_url, container_name=container_name, blob_name=blob_name, - snapshot=snapshot, credential=credential, **kwargs + account_url, + container_name=container_name, + blob_name=blob_name, + snapshot=snapshot, + credential=credential, + version_id=version_id, + audience=audience, + **kwargs ) @distributed_trace_async @@ -301,11 +346,13 @@ async def get_account_information(self, **kwargs: Any) -> Dict[str, str]: The keys in the returned dictionary include 'sku_name' and 'account_kind'. :returns: A dict of account information (SKU and account type). - :rtype: dict(str, str) + :rtype: Dict[str, str] """ try: - return cast(Dict[str, str], - await self._client.blob.get_account_info(cls=return_response_headers, **kwargs)) + return cast( + Dict[str, str], + await self._client.blob.get_account_info(cls=return_response_headers, **kwargs) + ) except HttpResponseError as error: process_storage_error(error) @@ -314,6 +361,25 @@ async def upload_blob_from_url( self, source_url: str, *, metadata: Optional[Dict[str, str]] = None, + overwrite: Optional[bool] = None, + include_source_blob_properties: Optional[bool] = None, + tags: Optional[Dict[str, str]] = None, + source_content_md5: Optional[bytearray] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + destination_lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + content_settings: Optional["ContentSettings"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + source_authorization: Optional[str] = None, **kwargs: Any ) -> Dict[str, Any]: """ @@ -332,7 +398,7 @@ async def upload_blob_from_url( https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot= https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken - :keyword dict(str, str) metadata: + :keyword Dict[str, str] metadata: Name-value pairs associated with the blob as metadata. :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. If True, upload_blob will overwrite the existing data. If set to False, the @@ -345,7 +411,7 @@ async def upload_blob_from_url( and tag values must be between 0 and 256 characters. Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9), space (' '), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_) - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword bytearray source_content_md5: Specify the md5 that is used to verify the integrity of the source bytes. :keyword ~datetime.datetime source_if_modified_since: @@ -415,12 +481,32 @@ async def upload_blob_from_url( :returns: Response from creating a new block blob for a given URL. :rtype: Dict[str, Any] """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_blob_from_url_options( source_url=source_url, metadata=metadata, - **kwargs) + overwrite=overwrite, + include_source_blob_properties=include_source_blob_properties, + tags=tags, + source_content_md5=source_content_md5, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + destination_lease=destination_lease, + timeout=timeout, + content_settings=content_settings, + cpk=cpk, + encryption_scope=encryption_scope, + standard_blob_tier=standard_blob_tier, + source_authorization=source_authorization, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.block_blob.put_blob_from_url(**options)) except HttpResponseError as error: @@ -445,7 +531,7 @@ async def upload_blob( should be supplied for optimal performance. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -455,7 +541,7 @@ async def upload_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. If True, upload_blob will overwrite the existing data. If set to False, the operation will fail with ResourceExistsError. The exception to the above is with Append @@ -563,7 +649,7 @@ async def upload_blob( multiple calls to the Azure service and the timeout will apply to each call individually. :returns: Blob-updated property dict (Etag and last modified) - :rtype: dict[str, Any] + :rtype: Dict[str, Any] .. admonition:: Example: @@ -1054,7 +1140,7 @@ async def set_blob_metadata( Dict containing name and value pairs. Each call to this operation replaces all existing metadata attached to the blob. To remove all metadata from the blob, call this operation with no metadata headers. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -1214,7 +1300,7 @@ async def create_page_blob( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :param ~azure.storage.blob.PremiumPageBlobTier premium_page_blob_tier: A page blob tier value to set the blob to. The tier correlates to the size of the blob and number of allowed IOPS. This is only applicable to page blobs on @@ -1228,7 +1314,7 @@ async def create_page_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword int sequence_number: Only for Page blobs. The sequence number is a user-controlled value that you can use to track requests. The value of the sequence number must be between 0 @@ -1286,7 +1372,7 @@ async def create_page_blob( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -1318,7 +1404,7 @@ async def create_append_blob( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -1328,7 +1414,7 @@ async def create_append_blob( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword ~azure.storage.blob.ImmutabilityPolicy immutability_policy: Specifies the immutability policy of a blob, blob snapshot or blob version. @@ -1382,7 +1468,7 @@ async def create_append_blob( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -1414,7 +1500,7 @@ async def create_snapshot( :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :keyword ~datetime.datetime if_modified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. @@ -1462,7 +1548,7 @@ async def create_snapshot( see `here `__. :returns: Blob-updated property dict (Snapshot ID, Etag, and last modified). - :rtype: dict[str, Any] + :rtype: Dict[str, Any] .. admonition:: Example: @@ -1536,7 +1622,7 @@ async def start_copy_from_url( source blob or file to the destination blob. If one or more name-value pairs are specified, the destination blob is created with the specified metadata, and metadata is not copied from the source blob or file. - :type metadata: dict(str, str) + :type metadata: Dict[str, str] :param bool incremental_copy: Copies the snapshot of the source page blob to a destination page blob. The snapshot is copied such that only the differential changes between @@ -1555,7 +1641,7 @@ async def start_copy_from_url( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) or Literal["COPY"] + :paramtype tags: Dict[str, str] or Literal["COPY"] :keyword ~azure.storage.blob.ImmutabilityPolicy immutability_policy: Specifies the immutability policy of a blob, blob snapshot or blob version. @@ -2023,7 +2109,7 @@ async def commit_block_list( language, disposition, md5, and cache control. :param metadata: Name-value pairs associated with the blob as metadata. - :type metadata: dict[str, str] + :type metadata: Dict[str, str] :keyword tags: Name-value pairs associated with the blob as tag. Tags are case-sensitive. The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, @@ -2033,7 +2119,7 @@ async def commit_block_list( .. versionadded:: 12.4.0 - :paramtype tags: dict(str, str) + :paramtype tags: Dict[str, str] :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -2103,7 +2189,7 @@ async def commit_block_list( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -2175,7 +2261,7 @@ async def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: A and tag values must be between 0 and 256 characters. Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9), space (' '), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_) - :type tags: dict(str, str) + :type tags: Dict[str, str] :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to delete. @@ -2307,7 +2393,7 @@ async def get_page_ranges( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(dict(str, str), list(dict(str, str)) + :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) """ warnings.warn( "get_page_ranges is deprecated, use list_page_ranges instead", @@ -2483,7 +2569,7 @@ async def get_page_range_diff_for_managed_disk( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(dict(str, str), list(dict(str, str)) + :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) """ options = _get_page_ranges_options( snapshot=self.snapshot, @@ -2546,7 +2632,7 @@ async def set_sequence_number( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ options = _set_sequence_number_options(sequence_number_action, sequence_number=sequence_number, **kwargs) try: @@ -2602,7 +2688,7 @@ async def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, da see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if kwargs.get('cpk') and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") @@ -2698,7 +2784,7 @@ async def upload_page( see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -2901,7 +2987,7 @@ async def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, see `here `__. :returns: Blob-updated property dict (Etag and last modified). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -2998,7 +3084,7 @@ async def append_block( see `here `__. :returns: Blob-updated property dict (Etag, last modified, append offset, committed block count). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) @@ -3171,7 +3257,7 @@ async def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime see `here `__. :returns: Blob-updated property dict (Etag, last modified, append offset, committed block count). - :rtype: dict(str, Any) + :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py index 8f76aa98c8cf..0b74e278a796 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py @@ -128,6 +128,9 @@ class BlobServiceClient( # type: ignore [misc] def __init__( self, account_url: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + # TODO **kwargs: Any ) -> None: kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) @@ -136,7 +139,7 @@ def __init__( self._query_str, credential = self._format_query_string(sas_token, credential) super(BlobServiceClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) - self._client._config.version = get_api_version(kwargs) # type: ignore [assignment] + self._client._config.version = get_api_version(api_version) # type: ignore [assignment] self._configure_encryption(kwargs) def _format_url(self, hostname): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py index 306e3acf5519..eeb97625131b 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py @@ -129,6 +129,9 @@ def __init__( self, account_url: str, container_name: str, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + *, + api_version: Optional[str] = None, + # TODO **kwargs: Any ) -> None: kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) @@ -139,7 +142,7 @@ def __init__( self._raw_credential = credential if credential else sas_token self._query_str, credential = self._format_query_string(sas_token, credential) super(ContainerClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) - self._api_version = get_api_version(kwargs) + self._api_version = get_api_version(api_version) self._client = self._build_generated_client() self._configure_encryption(kwargs) diff --git a/sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat b/sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat new file mode 100644 index 000000000000..e69de29bb2d1 From 58e2213b61f3e00b61db15b5cfe1ace6c82335dc Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 24 Mar 2025 16:11:29 -0400 Subject: [PATCH 02/43] type/pylint ignore errors formatting for _blob_service_client.py --- .../azure/storage/blob/_blob_service_client.py | 18 +++++++++--------- .../blob/aio/_blob_service_client_async.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py index 01b35c98bef4..749fac377db3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py @@ -243,7 +243,7 @@ def get_account_information(self, **kwargs: Any) -> Dict[str, str]: :caption: Getting account information for the blob service. """ try: - return self._client.service.get_account_info(cls=return_response_headers, **kwargs) # type: ignore + return self._client.service.get_account_info(cls=return_response_headers, **kwargs) # type: ignore except HttpResponseError as error: process_storage_error(error) @@ -287,7 +287,7 @@ def get_service_stats(self, **kwargs: Any) -> Dict[str, Any]: """ timeout = kwargs.pop('timeout', None) try: - stats = self._client.service.get_statistics( # type: ignore + stats = self._client.service.get_statistics( # type: ignore timeout=timeout, use_location=LocationMode.SECONDARY, **kwargs) return service_stats_deserialize(stats) except HttpResponseError as error: @@ -394,7 +394,7 @@ def set_service_properties( logging=analytics_logging, hour_metrics=hour_metrics, minute_metrics=minute_metrics, - cors=CorsRule._to_generated(cors), # pylint: disable=protected-access + cors=CorsRule._to_generated(cors), # pylint: disable=protected-access default_service_version=target_version, delete_retention_policy=delete_retention_policy, static_website=static_website @@ -651,7 +651,7 @@ def _rename_container(self, name: str, new_name: str, **kwargs: Any) -> Containe except AttributeError: kwargs['source_lease_id'] = lease try: - renamed_container._client.container.rename(name, **kwargs) # pylint: disable = protected-access + renamed_container._client.container.rename(name, **kwargs) # pylint: disable=protected-access return renamed_container except HttpResponseError as error: process_storage_error(error) @@ -688,7 +688,7 @@ def undelete_container( warnings.warn("`new_name` is no longer supported.", DeprecationWarning) container = self.get_container_client(new_name or deleted_container_name) try: - container._client.container.restore(deleted_container_name=deleted_container_name, # pylint: disable = protected-access + container._client.container.restore(deleted_container_name=deleted_container_name, # pylint: disable=protected-access deleted_container_version=deleted_container_version, timeout=kwargs.pop('timeout', None), **kwargs) return container @@ -721,8 +721,8 @@ def get_container_client(self, container: Union[ContainerProperties, str]) -> Co else: container_name = container _pipeline = Pipeline( - transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access - policies=self._pipeline._impl_policies # pylint: disable = protected-access + transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access + policies=self._pipeline._impl_policies # pylint: disable=protected-access ) return ContainerClient( self.url, container_name=container_name, @@ -779,8 +779,8 @@ def get_blob_client( else: container_name = container _pipeline = Pipeline( - transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access - policies=self._pipeline._impl_policies # pylint: disable = protected-access + transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access + policies=self._pipeline._impl_policies # pylint: disable=protected-access ) return BlobClient( self.url, container_name=container_name, blob_name=blob_name, snapshot=snapshot, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py index 0b74e278a796..0eb43d3a2d95 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py @@ -251,7 +251,7 @@ async def get_account_information(self, **kwargs: Any) -> Dict[str, str]: :caption: Getting account information for the blob service. """ try: - return await self._client.service.get_account_info(cls=return_response_headers, **kwargs) # type: ignore + return await self._client.service.get_account_info(cls=return_response_headers, **kwargs) # type: ignore except HttpResponseError as error: process_storage_error(error) @@ -295,7 +295,7 @@ async def get_service_stats(self, **kwargs: Any) -> Dict[str, Any]: """ timeout = kwargs.pop('timeout', None) try: - stats = await self._client.service.get_statistics( # type: ignore + stats = await self._client.service.get_statistics( # type: ignore timeout=timeout, use_location=LocationMode.SECONDARY, **kwargs) return service_stats_deserialize(stats) except HttpResponseError as error: @@ -402,7 +402,7 @@ async def set_service_properties( logging=analytics_logging, hour_metrics=hour_metrics, minute_metrics=minute_metrics, - cors=CorsRule._to_generated(cors), # pylint: disable=protected-access + cors=CorsRule._to_generated(cors), # pylint: disable=protected-access default_service_version=target_version, delete_retention_policy=delete_retention_policy, static_website=static_website @@ -659,7 +659,7 @@ async def _rename_container(self, name: str, new_name: str, **kwargs: Any) -> Co except AttributeError: kwargs['source_lease_id'] = lease try: - await renamed_container._client.container.rename(name, **kwargs) # pylint: disable = protected-access + await renamed_container._client.container.rename(name, **kwargs) # pylint: disable=protected-access return renamed_container except HttpResponseError as error: process_storage_error(error) @@ -696,7 +696,7 @@ async def undelete_container( warnings.warn("`new_name` is no longer supported.", DeprecationWarning) container = self.get_container_client(new_name or deleted_container_name) try: - await container._client.container.restore(deleted_container_name=deleted_container_name, # pylint: disable = protected-access + await container._client.container.restore(deleted_container_name=deleted_container_name, # pylint: disable=protected-access deleted_container_version=deleted_container_version, timeout=kwargs.pop('timeout', None), **kwargs) return container @@ -729,8 +729,8 @@ def get_container_client(self, container: Union[ContainerProperties, str]) -> Co else: container_name = container _pipeline = AsyncPipeline( - transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access - policies=self._pipeline._impl_policies #type: ignore [arg-type] # pylint: disable = protected-access + transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access + policies=self._pipeline._impl_policies #type: ignore [arg-type] # pylint: disable=protected-access ) return ContainerClient( self.url, container_name=container_name, @@ -789,9 +789,9 @@ def get_blob_client( else: container_name = container _pipeline = AsyncPipeline( - transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable = protected-access + transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access policies=cast(Iterable["AsyncHTTPPolicy"], - self._pipeline._impl_policies) # pylint: disable = protected-access + self._pipeline._impl_policies) # pylint: disable=protected-access ) return BlobClient( self.url, container_name=container_name, blob_name=blob_name, snapshot=snapshot, From e3d51a8d5dc9db3b1576b026d1a05664381129d4 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 24 Mar 2025 21:43:36 -0400 Subject: [PATCH 03/43] BlobClient needs integral default values for max/min configs --- .../azure/storage/blob/_blob_client.py | 14 +++++++------- .../azure/storage/blob/aio/_blob_client_async.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 6d14ad865884..fd6a6182db63 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -170,13 +170,13 @@ def __init__( api_version: Optional[str] = None, secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, - audience: Optional[int] = None, - max_block_size: Optional[int] = None, - max_page_size: Optional[int] = None, - max_chunk_get_size: Optional[int] = None, - max_single_put_size: Optional[int] = None, - max_single_get_size: Optional[int] = None, - min_large_block_upload_threshold: Optional[int] = None, + audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> None: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 917d0d9a12b7..ca7eab8a3cb8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -165,13 +165,13 @@ def __init__( api_version: Optional[str] = None, secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, - audience: Optional[int] = None, - max_block_size: Optional[int] = None, - max_page_size: Optional[int] = None, - max_chunk_get_size: Optional[int] = None, - max_single_put_size: Optional[int] = None, - max_single_get_size: Optional[int] = None, - min_large_block_upload_threshold: Optional[int] = None, + audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> None: From 978a151f17daa53b7368e32025fd9259c040427a Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Tue, 25 Mar 2025 17:14:35 -0400 Subject: [PATCH 04/43] [WIP] upload_blob --- .../azure/storage/blob/_blob_client.py | 33 ++++++++++++++++--- .../storage/blob/aio/_blob_client_async.py | 31 ++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index fd6a6182db63..8a992500cb7c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -9,7 +9,7 @@ from datetime import datetime from functools import partial from typing import ( - Any, AnyStr, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, + Any, AnyStr, Callable, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -501,7 +501,6 @@ def upload_blob_from_url( source_content_md5=source_content_md5, source_if_modified_since=source_if_modified_since, source_if_unmodified_since=source_if_unmodified_since, - source_etag=source_etag, source_match_condition=source_match_condition, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, @@ -527,6 +526,28 @@ def upload_blob( blob_type: Union[str, BlobType] = BlobType.BLOCKBLOB, length: Optional[int] = None, metadata: Optional[Dict[str, str]] = None, + *, + tags: Optional[Dict[str, str]] = None, + overwrite: Optional[bool] = None, + content_settings: Optional["ContentSettings"] = None, + validate_content: Optional[bool] = None, + lease: Optional[BlobLeaseClient] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_conditions: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + maxsize_condition: Optional[int] = None, + max_concurrency: Optional[int] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + encoding: Optional[str] = None, + progress_hook: Optional[Callable[[int, Optional[int]], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new blob from a data source with automatic chunking. @@ -669,7 +690,7 @@ def upload_blob( """ if self.require_encryption and not self.key_encryption_key: raise ValueError("Encryption required but no key was provided.") - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_blob_options( data=data, @@ -685,7 +706,8 @@ def upload_blob( config=self._config, sdk_moniker=self._sdk_moniker, client=self._client, - **kwargs) + **kwargs + ) if blob_type == BlobType.BlockBlob: return upload_block_blob(**options) if blob_type == BlobType.PageBlob: @@ -834,7 +856,8 @@ def download_blob( config=self._config, sdk_moniker=self._sdk_moniker, client=self._client, - **kwargs) + **kwargs + ) return StorageStreamDownloader(**options) @distributed_trace diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index ca7eab8a3cb8..69e477e785c3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -9,7 +9,7 @@ from datetime import datetime from functools import partial from typing import ( - Any, AnyStr, AsyncIterable, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, + Any, AnyStr, AsyncIterable, Callable, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -518,6 +518,27 @@ async def upload_blob( blob_type: Union[str, BlobType] = BlobType.BLOCKBLOB, length: Optional[int] = None, metadata: Optional[Dict[str, str]] = None, + tags: Optional[Dict[str, str]] = None, + overwrite: Optional[bool] = None, + content_settings: Optional["ContentSettings"] = None, + validate_content: Optional[bool] = None, + lease: Optional[BlobLeaseClient] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_conditions: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + maxsize_condition: Optional[int] = None, + max_concurrency: Optional[int] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + encoding: Optional[str] = None, + progress_hook: Optional[Callable[[int, Optional[int]], Awaitable[None]]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new blob from a data source with automatic chunking. @@ -662,7 +683,7 @@ async def upload_blob( """ if self.require_encryption and not self.key_encryption_key: raise ValueError("Encryption required but no key was provided.") - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_blob_options( data=data, @@ -678,7 +699,8 @@ async def upload_blob( config=self._config, sdk_moniker=self._sdk_moniker, client=self._client, - **kwargs) + **kwargs + ) if blob_type == BlobType.BlockBlob: return cast(Dict[str, Any], await upload_block_blob(**options)) if blob_type == BlobType.PageBlob: @@ -827,7 +849,8 @@ async def download_blob( config=self._config, sdk_moniker=self._sdk_moniker, client=self._client, - **kwargs) + **kwargs + ) downloader = StorageStreamDownloader(**options) await downloader._setup() # pylint: disable=protected-access return downloader From f7345a92a3b83238fa13d04da63e36328867d288 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 16:00:18 -0400 Subject: [PATCH 05/43] upload_blob --- .../azure/storage/blob/_blob_client.py | 25 ++++++++++++++++++- .../storage/blob/_blob_client_helpers.py | 2 +- .../storage/blob/aio/_blob_client_async.py | 24 +++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 8a992500cb7c..3a9650bb6b0a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -9,7 +9,8 @@ from datetime import datetime from functools import partial from typing import ( - Any, AnyStr, Callable, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, + Any, AnyStr, Callable, cast, Dict, IO, Iterable, + List, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -501,6 +502,7 @@ def upload_blob_from_url( source_content_md5=source_content_md5, source_if_modified_since=source_if_modified_since, source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, source_match_condition=source_match_condition, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, @@ -697,6 +699,27 @@ def upload_blob( blob_type=blob_type, length=length, metadata=metadata, + tags=tags, + overwrite=overwrite, + content_settings=content_settings, + validate_content=validate_content, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_conditions=match_conditions, + if_tags_match_condition=if_tags_match_condition, + premium_page_blob_tier=premium_page_blob_tier, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + standard_blob_tier=standard_blob_tier, + maxsize_condition=maxsize_condition, + max_concurrency=max_concurrency, + cpk=cpk, + encryption_scope=encryption_scope, + encoding=encoding, + progress_hook=progress_hook, + timeout=timeout, encryption_options={ 'required': self.require_encryption, 'version': self.encryption_version, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index a04f0ea02525..904f2be07026 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -228,7 +228,7 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A 'cpk_scope_info': get_cpk_scope_info(kwargs), 'headers': headers, } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) if not overwrite and not _any_conditions(**options): options['modified_access_conditions'].if_none_match = '*' return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 69e477e785c3..2f28c582a1d3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -9,7 +9,8 @@ from datetime import datetime from functools import partial from typing import ( - Any, AnyStr, AsyncIterable, Callable, cast, Dict, IO, Iterable, List, Optional, overload, Tuple, Union, + Any, AnyStr, AsyncIterable, Awaitable, Callable, cast, Dict, + IO, Iterable, List, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -690,6 +691,27 @@ async def upload_blob( blob_type=blob_type, length=length, metadata=metadata, + tags=tags, + overwrite=overwrite, + content_settings=content_settings, + validate_content=validate_content, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_conditions=match_conditions, + if_tags_match_condition=if_tags_match_condition, + premium_page_blob_tier=premium_page_blob_tier, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + standard_blob_tier=standard_blob_tier, + maxsize_condition=maxsize_condition, + max_concurrency=max_concurrency, + cpk=cpk, + encryption_scope=encryption_scope, + encoding=encoding, + progress_hook=progress_hook, + timeout=timeout, encryption_options={ 'required': self.require_encryption, 'version': self.encryption_version, From a7a5fa0bc6a5fb91deec02281f999663db4b2086 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 16:04:37 -0400 Subject: [PATCH 06/43] Removed extra file --- ...g_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat diff --git a/sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat b/sdk/storage/azure-storage-blob/tests/file_with_existing_file_overwrite.temp.250ab16d-214e-4844-a12c-feecb1af04c1.dat deleted file mode 100644 index e69de29bb2d1..000000000000 From bc46497ba5bfe773d19f7d5eeeecbc6074bf7e8a Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 16:20:18 -0400 Subject: [PATCH 07/43] download_blob --- .../azure/storage/blob/_blob_client.py | 53 +++++++++++++++++-- .../storage/blob/_blob_client_helpers.py | 15 ++++-- .../storage/blob/aio/_blob_client_async.py | 51 +++++++++++++++++- 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 3a9650bb6b0a..6ea7d011dcad 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -743,6 +743,18 @@ def download_blob( length: Optional[int] = None, *, encoding: str, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + progress_hook: Optional[Callable[[int, int], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[str]: ... @@ -753,6 +765,18 @@ def download_blob( length: Optional[int] = None, *, encoding: None = None, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + progress_hook: Optional[Callable[[int, int], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[bytes]: ... @@ -762,7 +786,19 @@ def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: Union[str, None] = None, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + encoding: Optional[str] = None, + progress_hook: Optional[Callable[[int, int], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Union[StorageStreamDownloader[str], StorageStreamDownloader[bytes]]: """Downloads a blob to the StorageStreamDownloader. The readall() method must @@ -861,15 +897,26 @@ def download_blob( raise ValueError("Encryption required but no key was provided.") if length is not None and offset is None: raise ValueError("Offset value must not be None if length is set.") - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _download_blob_options( blob_name=self.blob_name, container_name=self.container_name, - version_id=get_version_id(self.version_id, kwargs), + version_id=version_id or self.version_id, offset=offset, length=length, encoding=encoding, + validate_content=validate_content, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + max_concurrency=max_concurrency, + progress_hook=progress_hook, + timeout=timeout, encryption_options={ 'required': self.require_encryption, 'version': self.encryption_version, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 904f2be07026..59ff259b724f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -283,8 +283,11 @@ def _download_blob_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) # Add feature flag to user agent for encryption if encryption_options['key'] or encryption_options['resolver']: @@ -292,7 +295,8 @@ def _download_blob_options( config.user_agent_policy.user_agent, sdk_moniker, encryption_options['version'], - kwargs) + kwargs + ) options = { 'clients': client, @@ -309,11 +313,12 @@ def _download_blob_options( 'modified_access_conditions': mod_conditions, 'cpk_info': cpk_info, 'download_cls': kwargs.pop('cls', None) or deserialize_blob_stream, - 'max_concurrency':kwargs.pop('max_concurrency', 1), + 'max_concurrency': kwargs.pop('max_concurrency', 1), 'encoding': encoding, 'timeout': kwargs.pop('timeout', None), 'name': blob_name, - 'container': container_name} + 'container': container_name + } options.update(kwargs) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 2f28c582a1d3..1eeeaa918a69 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -735,6 +735,18 @@ async def download_blob( length: Optional[int] = None, *, encoding: str, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + progress_hook: Optional[Callable[[int, int], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[str]: ... @@ -745,6 +757,18 @@ async def download_blob( length: Optional[int] = None, *, encoding: None = None, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + progress_hook: Optional[Callable[[int, int], None]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[bytes]: ... @@ -754,7 +778,19 @@ async def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: Union[str, None] = None, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + max_concurrency: Optional[int] = None, + encoding: Optional[str] = None, + progress_hook: Optional[Callable[[int, int], Awaitable[None]]] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Union[StorageStreamDownloader[str], StorageStreamDownloader[bytes]]: """Downloads a blob to the StorageStreamDownloader. The readall() method must @@ -858,10 +894,21 @@ async def download_blob( options = _download_blob_options( blob_name=self.blob_name, container_name=self.container_name, - version_id=get_version_id(self.version_id, kwargs), + version_id=version_id or self.version_id, offset=offset, length=length, encoding=encoding, + validate_content=validate_content, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + max_concurrency=max_concurrency, + progress_hook=progress_hook, + timeout=timeout, encryption_options={ 'required': self.require_encryption, 'version': self.encryption_version, From 1333854fc2536ab400fa20c450eccef79db2e50c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 16:21:53 -0400 Subject: [PATCH 08/43] Some formatting... --- .../azure/storage/blob/_blob_client_helpers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 59ff259b724f..2878340201f7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -209,8 +209,11 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + ncryption_algorithm=cpk.algorithm + ) options = { 'copy_source_authorization': source_authorization, From 5e140d71bddd36723bb1884876d42f4546c567c3 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 16:58:57 -0400 Subject: [PATCH 09/43] pop etag if None --- .../azure/storage/blob/_blob_client_helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 2878340201f7..b396a5baf34d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -142,6 +142,9 @@ def _upload_blob_options( # pylint:disable=too-many-statements encryption_algorithm=cpk.algorithm) kwargs['cpk_info'] = cpk_info + if kwargs['etag'] is None: + kwargs.pop('etag') + headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) kwargs['lease_access_conditions'] = get_access_conditions(kwargs.pop('lease', None)) @@ -171,7 +174,8 @@ def _upload_blob_options( # pylint:disable=too-many-statements config.user_agent_policy.user_agent, sdk_moniker, encryption_options['version'], - kwargs) + kwargs + ) if blob_type == BlobType.BlockBlob: kwargs['client'] = client.block_blob @@ -292,6 +296,9 @@ def _download_blob_options( encryption_algorithm=cpk.algorithm ) + if kwargs['etag'] is None: + kwargs.pop('etag') + # Add feature flag to user agent for encryption if encryption_options['key'] or encryption_options['resolver']: modify_user_agent_for_encryption( From 0e0d185892a0c39c6489dc3b70370cb0a8312903 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 17:38:24 -0400 Subject: [PATCH 10/43] match condition typo --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 6ea7d011dcad..36fd15e050ef 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -537,7 +537,7 @@ def upload_blob( if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, - match_conditions: Optional["MatchConditions"] = None, + match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, immutability_policy: Optional["ImmutabilityPolicy"] = None, @@ -707,7 +707,7 @@ def upload_blob( if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, etag=etag, - match_conditions=match_conditions, + match_condition=match_condition, if_tags_match_condition=if_tags_match_condition, premium_page_blob_tier=premium_page_blob_tier, immutability_policy=immutability_policy, From abedceae41eac1d532b15968bdd4128ca7280ec3 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 17:58:07 -0400 Subject: [PATCH 11/43] updated default values for upload/download apis --- .../azure/storage/blob/_blob_client.py | 20 +++++++++---------- .../storage/blob/_blob_client_helpers.py | 14 ++++--------- .../storage/blob/aio/_blob_client_async.py | 20 +++++++++---------- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 36fd15e050ef..3684735f1004 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -530,9 +530,9 @@ def upload_blob( metadata: Optional[Dict[str, str]] = None, *, tags: Optional[Dict[str, str]] = None, - overwrite: Optional[bool] = None, + overwrite: bool = False, content_settings: Optional["ContentSettings"] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[BlobLeaseClient] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -544,10 +544,10 @@ def upload_blob( legal_hold: Optional[bool] = None, standard_blob_tier: Optional["StandardBlobTier"] = None, maxsize_condition: Optional[int] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, cpk: Optional["CustomerProvidedEncryptionKey"] = None, encryption_scope: Optional[str] = None, - encoding: Optional[str] = None, + encoding: str = 'UTF-8', progress_hook: Optional[Callable[[int, Optional[int]], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -744,7 +744,7 @@ def download_blob( *, encoding: str, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -752,7 +752,7 @@ def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -766,7 +766,7 @@ def download_blob( *, encoding: None = None, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -774,7 +774,7 @@ def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -787,7 +787,7 @@ def download_blob( length: Optional[int] = None, *, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -795,7 +795,7 @@ def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, encoding: Optional[str] = None, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index b396a5baf34d..1b709a2479eb 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -142,9 +142,6 @@ def _upload_blob_options( # pylint:disable=too-many-statements encryption_algorithm=cpk.algorithm) kwargs['cpk_info'] = cpk_info - if kwargs['etag'] is None: - kwargs.pop('etag') - headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) kwargs['lease_access_conditions'] = get_access_conditions(kwargs.pop('lease', None)) @@ -190,7 +187,7 @@ def _upload_blob_options( # pylint:disable=too-many-statements kwargs['client'] = client.append_blob else: raise ValueError(f"Unsupported BlobType: {blob_type}") - return kwargs + return {k: v for k, v in kwargs.items() if v is not None} def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, Any]: metadata = kwargs.pop('metadata', None) @@ -235,10 +232,10 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A 'cpk_scope_info': get_cpk_scope_info(kwargs), 'headers': headers, } - options.update({k: v for k, v in kwargs.items() if v is not None}) + options.update(kwargs) if not overwrite and not _any_conditions(**options): options['modified_access_conditions'].if_none_match = '*' - return options + return {k: v for k, v in options.items() if v is not None} def _download_blob_options( blob_name: str, @@ -296,9 +293,6 @@ def _download_blob_options( encryption_algorithm=cpk.algorithm ) - if kwargs['etag'] is None: - kwargs.pop('etag') - # Add feature flag to user agent for encryption if encryption_options['key'] or encryption_options['resolver']: modify_user_agent_for_encryption( @@ -330,7 +324,7 @@ def _download_blob_options( 'container': container_name } options.update(kwargs) - return options + return {k: v for k, v in options.items() if v is not None} def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwargs: Any ) -> Tuple[Dict[str, Any], str]: delimiter = '\n' diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 1eeeaa918a69..382efffaf8b6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -520,9 +520,9 @@ async def upload_blob( length: Optional[int] = None, metadata: Optional[Dict[str, str]] = None, tags: Optional[Dict[str, str]] = None, - overwrite: Optional[bool] = None, + overwrite: bool = False, content_settings: Optional["ContentSettings"] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[BlobLeaseClient] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -534,10 +534,10 @@ async def upload_blob( legal_hold: Optional[bool] = None, standard_blob_tier: Optional["StandardBlobTier"] = None, maxsize_condition: Optional[int] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, cpk: Optional["CustomerProvidedEncryptionKey"] = None, encryption_scope: Optional[str] = None, - encoding: Optional[str] = None, + encoding: str = 'UTF-8', progress_hook: Optional[Callable[[int, Optional[int]], Awaitable[None]]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -736,7 +736,7 @@ async def download_blob( *, encoding: str, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -744,7 +744,7 @@ async def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -758,7 +758,7 @@ async def download_blob( *, encoding: None = None, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -766,7 +766,7 @@ async def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -779,7 +779,7 @@ async def download_blob( length: Optional[int] = None, *, version_id: Optional[str] = None, - validate_content: Optional[bool] = None, + validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, @@ -787,7 +787,7 @@ async def download_blob( match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, - max_concurrency: Optional[int] = None, + max_concurrency: int = 1, encoding: Optional[str] = None, progress_hook: Optional[Callable[[int, int], Awaitable[None]]] = None, timeout: Optional[int] = None, From 6061a3ab58c419c839aefae0818711721dfff902 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 20:14:28 -0400 Subject: [PATCH 12/43] include_source_blob_properties in upload_blob_from_url should be defaulted to True --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 2 +- .../azure/storage/blob/aio/_blob_client_async.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 3684735f1004..36d12cc9986f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -372,7 +372,7 @@ def upload_blob_from_url( *, metadata: Optional[Dict[str, str]] = None, overwrite: Optional[bool] = None, - include_source_blob_properties: Optional[bool] = None, + include_source_blob_properties: bool = True, tags: Optional[Dict[str, str]] = None, source_content_md5: Optional[bytearray] = None, source_if_modified_since: Optional[datetime] = None, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 382efffaf8b6..7c9e23b02bc0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -363,7 +363,7 @@ async def upload_blob_from_url( *, metadata: Optional[Dict[str, str]] = None, overwrite: Optional[bool] = None, - include_source_blob_properties: Optional[bool] = None, + include_source_blob_properties: bool = True, tags: Optional[Dict[str, str]] = None, source_content_md5: Optional[bytearray] = None, source_if_modified_since: Optional[datetime] = None, From e4df2ecf9bbe47bfc5165914bd619a24b59479bb Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 26 Mar 2025 20:32:28 -0400 Subject: [PATCH 13/43] missing encryption_algorithm --- .../azure/storage/blob/_blob_client_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 1b709a2479eb..86b0ce3c5a14 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -213,7 +213,7 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A cpk_info = CpkInfo( encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - ncryption_algorithm=cpk.algorithm + encryption_algorithm=cpk.algorithm ) options = { @@ -232,10 +232,10 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A 'cpk_scope_info': get_cpk_scope_info(kwargs), 'headers': headers, } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) if not overwrite and not _any_conditions(**options): options['modified_access_conditions'].if_none_match = '*' - return {k: v for k, v in options.items() if v is not None} + return options def _download_blob_options( blob_name: str, @@ -323,8 +323,8 @@ def _download_blob_options( 'name': blob_name, 'container': container_name } - options.update(kwargs) - return {k: v for k, v in options.items() if v is not None} + options.update({k: v for k, v in kwargs.items() if v is not None}) + return options def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwargs: Any ) -> Tuple[Dict[str, Any], str]: delimiter = '\n' From 688c71c0b702742890070c858c8543c477d639ff Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 00:52:11 -0400 Subject: [PATCH 14/43] suppressed too-many-locals pylint error --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 2 +- .../azure/storage/blob/aio/_blob_client_async.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 36d12cc9986f..f6b5cb97a5b6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only +# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only, too-many-locals import warnings from datetime import datetime diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 7c9e23b02bc0..478686e48bd0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only +# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only, too-many-locals import warnings from datetime import datetime From 5ef4abb074c13b8242458cfd64a72bbfdadfbbde Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 12:01:06 -0400 Subject: [PATCH 15/43] Fixed overloaded signature for download_blob API --- .../azure/storage/blob/_blob_client.py | 4 ++-- .../azure/storage/blob/aio/_blob_client_async.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index f6b5cb97a5b6..be3674da512f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -742,7 +742,6 @@ def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: str, version_id: Optional[str] = None, validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, @@ -753,6 +752,7 @@ def download_blob( if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, max_concurrency: int = 1, + encoding: str, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -764,7 +764,6 @@ def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: None = None, version_id: Optional[str] = None, validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, @@ -775,6 +774,7 @@ def download_blob( if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, max_concurrency: int = 1, + encoding: None = None, progress_hook: Optional[Callable[[int, int], None]] = None, timeout: Optional[int] = None, **kwargs: Any diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 478686e48bd0..d2e3becb87ef 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -77,9 +77,11 @@ from .._shared.response_handlers import process_storage_error, return_response_headers if TYPE_CHECKING: + from azure.core import MatchConditions from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential from azure.core.credentials_async import AsyncTokenCredential from azure.core.pipeline.policies import AsyncHTTPPolicy + from azure.storage.blob import CustomerProvidedEncryptionKey from azure.storage.blob.aio import ContainerClient from .._models import ( ContentSettings, @@ -734,7 +736,6 @@ async def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: str, version_id: Optional[str] = None, validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, @@ -745,7 +746,8 @@ async def download_blob( if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, max_concurrency: int = 1, - progress_hook: Optional[Callable[[int, int], None]] = None, + encoding: str, + progress_hook: Optional[Callable[[int, int], Awaitable[None]]] = None, timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[str]: @@ -756,7 +758,6 @@ async def download_blob( self, offset: Optional[int] = None, length: Optional[int] = None, *, - encoding: None = None, version_id: Optional[str] = None, validate_content: bool = False, lease: Optional[Union[BlobLeaseClient, str]] = None, @@ -767,7 +768,8 @@ async def download_blob( if_tags_match_condition: Optional[str] = None, cpk: Optional["CustomerProvidedEncryptionKey"] = None, max_concurrency: int = 1, - progress_hook: Optional[Callable[[int, int], None]] = None, + encoding: None = None, + progress_hook: Optional[Callable[[int, int], Awaitable[None]]] = None, timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader[bytes]: From 83fef4bb1f321d436ed268c01a06dcaebb29319d Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 14:58:44 -0400 Subject: [PATCH 16/43] delete_blob --- .../azure/storage/blob/_blob_client.py | 26 ++++++++++++++++--- .../storage/blob/_blob_client_helpers.py | 5 ++-- .../azure/storage/blob/_serialize.py | 12 +++++---- .../storage/blob/aio/_blob_client_async.py | 26 ++++++++++++++++--- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index be3674da512f..d4744aa2f7e3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1030,7 +1030,19 @@ def query_blob(self, query_expression: str, **kwargs: Any) -> BlobQueryReader: error_cls=error_cls) @distributed_trace - def delete_blob(self, delete_snapshots: Optional[str] = None, **kwargs: Any) -> None: + def delete_blob( + self, delete_snapshots: Optional[str] = None, + *, + version_id: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """Marks the specified blob for deletion. The blob is later deleted during garbage collection. @@ -1103,9 +1115,17 @@ def delete_blob(self, delete_snapshots: Optional[str] = None, **kwargs: Any) -> """ options = _delete_blob_options( snapshot=self.snapshot, - version_id=get_version_id(self.version_id, kwargs), + version_id=version_id or self.version_id, delete_snapshots=delete_snapshots, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: self._client.blob.delete(**options) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 86b0ce3c5a14..62c98ab04ba2 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -398,8 +398,9 @@ def _generic_delete_blob_options(delete_snapshots: Optional[str] = None, **kwarg 'snapshot': kwargs.pop('snapshot', None), # this is added for delete_blobs 'delete_snapshots': delete_snapshots or None, 'lease_access_conditions': access_conditions, - 'modified_access_conditions': mod_conditions} - options.update(kwargs) + 'modified_access_conditions': mod_conditions + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options def _delete_blob_options( diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py index e6e9f163aaf0..56009ed68141 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py @@ -91,11 +91,13 @@ def _get_match_headers( def get_access_conditions(lease: Optional[Union["BlobLeaseClient", str]]) -> Optional[LeaseAccessConditions]: - try: - lease_id = lease.id # type: ignore - except AttributeError: - lease_id = lease # type: ignore - return LeaseAccessConditions(lease_id=lease_id) if lease_id else None + if lease is None: + return None + if hasattr(lease, "id"): + lease_id = lease.id # type: ignore + else: + lease_id = lease # type: ignore + return LeaseAccessConditions(lease_id=lease_id) def get_modify_conditions(kwargs: Dict[str, Any]) -> ModifiedAccessConditions: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index d2e3becb87ef..5bbd0c72c0bf 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -927,7 +927,19 @@ async def download_blob( return downloader @distributed_trace_async - async def delete_blob(self, delete_snapshots: Optional[str] = None, **kwargs: Any) -> None: + async def delete_blob( + self, delete_snapshots: Optional[str] = None, + *, + version_id: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional[MatchConditions] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """Marks the specified blob for deletion. The blob is later deleted during garbage collection. @@ -1000,9 +1012,17 @@ async def delete_blob(self, delete_snapshots: Optional[str] = None, **kwargs: An """ options = _delete_blob_options( snapshot=self.snapshot, - version_id=get_version_id(self.version_id, kwargs), + version_id=version_id or self.version_id, delete_snapshots=delete_snapshots, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: await self._client.blob.delete(**options) except HttpResponseError as error: From f93bc36c8da56337e51e6275454fdcf64e56e7bf Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 15:43:15 -0400 Subject: [PATCH 17/43] query_blob --- .../azure/storage/blob/_blob_client.py | 65 +++++++++++++++---- .../storage/blob/_blob_client_helpers.py | 4 +- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index d4744aa2f7e3..824525fd96fc 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -81,8 +81,16 @@ if TYPE_CHECKING: from azure.core import MatchConditions from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential, TokenCredential - from azure.storage.blob import ContainerClient, CustomerProvidedEncryptionKey + from azure.storage.blob import ( + ContainerClient, + CustomerProvidedEncryptionKey, + DelimitedTextDialect, + DelimitedJsonDialect, + QuickQueryDialect, + ArrowDialect +) from ._models import ( + BlobQueryError, ContentSettings, ImmutabilityPolicy, PremiumPageBlobTier, @@ -931,7 +939,22 @@ def download_blob( return StorageStreamDownloader(**options) @distributed_trace - def query_blob(self, query_expression: str, **kwargs: Any) -> BlobQueryReader: + def query_blob( + self, query_expression: str, + *, + on_error: Optional[Callable[["BlobQueryError"], None]] = None, + blob_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", str]] = None, + output_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", List["ArrowDialect"], str]] = None, # pylint: disable=line-too-long + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> BlobQueryReader: """Enables users to select/project on blob/or blob snapshot data by providing simple query expressions. This operations returns a BlobQueryReader, users need to use readall() or readinto() to get query data. @@ -949,16 +972,23 @@ def query_blob(self, query_expression: str, **kwargs: Any) -> BlobQueryReader: .. note:: "ParquetDialect" is in preview, so some features may not work as intended. - :paramtype blob_format: ~azure.storage.blob.DelimitedTextDialect or ~azure.storage.blob.DelimitedJsonDialect - or ~azure.storage.blob.QuickQueryDialect or str + :paramtype blob_format: + ~azure.storage.blob.DelimitedTextDialect or + ~azure.storage.blob.DelimitedJsonDialect or + ~azure.storage.blob.QuickQueryDialect or + str :keyword output_format: Optional. Defines the output serialization for the data stream. By default the data will be returned as it is represented in the blob (Parquet formats default to DelimitedTextDialect). By providing an output format, the blob data will be reformatted according to that profile. This value can be a DelimitedTextDialect or a DelimitedJsonDialect or ArrowDialect. These dialects can be passed through their respective classes, the QuickQueryDialect enum or as a string - :paramtype output_format: ~azure.storage.blob.DelimitedTextDialect or ~azure.storage.blob.DelimitedJsonDialect - or List[~azure.storage.blob.ArrowDialect] or ~azure.storage.blob.QuickQueryDialect or str + :paramtype output_format: + ~azure.storage.blob.DelimitedTextDialect or + ~azure.storage.blob.DelimitedJsonDialect or + List[~azure.storage.blob.ArrowDialect] or + ~azure.storage.blob.QuickQueryDialect or + str :keyword lease: Required if the blob has an active lease. Value can be a BlobLeaseClient object or the lease ID as a string. @@ -1009,12 +1039,24 @@ def query_blob(self, query_expression: str, **kwargs: Any) -> BlobQueryReader: :dedent: 4 :caption: select/project on blob/or blob snapshot data by providing simple query expressions. """ - errors = kwargs.pop("on_error", None) error_cls = kwargs.pop("error_cls", BlobQueryError) encoding = kwargs.pop("encoding", None) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options, delimiter = _quick_query_options(self.snapshot, query_expression, **kwargs) + options, delimiter = _quick_query_options( + self.snapshot, + query_expression, + blob_format=blob_format, + output_format=output_format, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: headers, raw_response_body = self._client.blob.query(**options) except HttpResponseError as error: @@ -1022,12 +1064,13 @@ def query_blob(self, query_expression: str, **kwargs: Any) -> BlobQueryReader: return BlobQueryReader( name=self.blob_name, container=self.container_name, - errors=errors, + errors=on_error, record_delimiter=delimiter, encoding=encoding, headers=headers, response=raw_response_body, - error_cls=error_cls) + error_cls=error_cls + ) @distributed_trace def delete_blob( diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 62c98ab04ba2..bb77494f0361 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -326,7 +326,7 @@ def _download_blob_options( options.update({k: v for k, v in kwargs.items() if v is not None}) return options -def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwargs: Any ) -> Tuple[Dict[str, Any], str]: +def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwargs: Any) -> Tuple[Dict[str, Any], str]: delimiter = '\n' input_format = kwargs.pop('blob_format', None) if input_format == QuickQueryDialect.DelimitedJson: @@ -385,7 +385,7 @@ def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwarg 'timeout': kwargs.pop('timeout', None), 'cls': return_headers_and_deserialized, } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options, delimiter def _generic_delete_blob_options(delete_snapshots: Optional[str] = None, **kwargs: Any) -> Dict[str, Any]: From 683fc8f1dca8ea45845fdfea4d301125780b39a7 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 15:51:48 -0400 Subject: [PATCH 18/43] Fixed annotation errors for delete_blob --- .../azure/storage/blob/aio/_blob_client_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 5bbd0c72c0bf..da835e3a1ad3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -935,7 +935,7 @@ async def delete_blob( if_modified_since: Optional[datetime] = None, if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, - match_condition: Optional[MatchConditions] = None, + match_condition: Optional["MatchConditions"] = None, if_tags_match_condition: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any From 23fbecadfab2140f0ad941547e8c5cc330584c4c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 27 Mar 2025 16:31:03 -0400 Subject: [PATCH 19/43] undelete_blob, get_blob_properties, exists --- .../azure/storage/blob/_blob_client.py | 67 +++++++++++-------- .../storage/blob/_blob_client_helpers.py | 53 +++++++++++++++ .../storage/blob/aio/_blob_client_async.py | 63 ++++++++++------- 3 files changed, 131 insertions(+), 52 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 824525fd96fc..223f59aae8a1 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -32,6 +32,7 @@ _download_blob_options, _format_url, _from_blob_url, + _get_blob_properties_options, _get_blob_tags_options, _get_block_list_result, _get_page_ranges_options, @@ -60,9 +61,8 @@ from ._download import StorageStreamDownloader from ._encryption import StorageEncryptionMixin, _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from ._generated import AzureBlobStorage -from ._generated.models import CpkInfo from ._lease import BlobLeaseClient -from ._models import BlobBlock, BlobProperties, BlobQueryError, BlobType, PageRange, PageRangePaged +from ._models import BlobBlock, BlobProperties, BlobType, PageRange, PageRangePaged from ._quick_query_helper import BlobQueryReader from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper from ._shared.response_handlers import process_storage_error, return_response_headers @@ -1054,6 +1054,7 @@ def query_blob( etag=etag, match_condition=match_condition, if_tags_match_condition=if_tags_match_condition, + cpk=cpk, timeout=timeout, **kwargs ) @@ -1175,7 +1176,7 @@ def delete_blob( process_storage_error(error) @distributed_trace - def undelete_blob(self, **kwargs: Any) -> None: + def undelete_blob(self, *, timeout: Optional[int] = None, **kwargs: Any) -> None: """Restores soft-deleted blobs or snapshots. Operation will only be successful if used within the specified number of days @@ -1203,12 +1204,12 @@ def undelete_blob(self, **kwargs: Any) -> None: :caption: Undeleting a blob. """ try: - self._client.blob.undelete(timeout=kwargs.pop('timeout', None), **kwargs) + self._client.blob.undelete(timeout=timeout, **kwargs) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def exists(self, **kwargs: Any) -> bool: + def exists(self, *, version_id: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any) -> bool: """ Returns True if a blob exists with the defined parameters, and returns False otherwise. @@ -1225,12 +1226,13 @@ def exists(self, **kwargs: Any) -> bool: :returns: boolean :rtype: bool """ - version_id = get_version_id(self.version_id, kwargs) try: self._client.blob.get_properties( snapshot=self.snapshot, - version_id=version_id, - **kwargs) + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + ) return True # Encrypted with CPK except ResourceExistsError: @@ -1242,7 +1244,19 @@ def exists(self, **kwargs: Any) -> bool: return False @distributed_trace - def get_blob_properties(self, **kwargs: Any) -> BlobProperties: + def get_blob_properties( + self, *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + version_id: Optional[str] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> BlobProperties: """Returns all user-defined metadata, standard HTTP properties, and system properties for the blob. It does not return the content of the blob. @@ -1304,30 +1318,29 @@ def get_blob_properties(self, **kwargs: Any) -> BlobProperties: :dedent: 8 :caption: Getting the properties for a blob. """ - # TODO: extract this out as _get_blob_properties_options - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) - version_id = get_version_id(self.version_id, kwargs) - cpk = kwargs.pop('cpk', None) - cpk_info = None - if cpk: - if self.scheme.lower() != 'https': - raise ValueError("Customer provided encryption key must be used over HTTPS.") - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + if cpk and self.scheme.lower() != 'https': + raise ValueError("Customer provided encryption key must be used over HTTPS.") + options = _get_blob_properties_options( + lease=lease, + version_id=version_id or self.version_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + snapshot=self.snapshot, + timeout=timeout, + **kwargs + ) try: cls_method = kwargs.pop('cls', None) if cls_method: kwargs['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) blob_props = cast(BlobProperties, self._client.blob.get_properties( - timeout=kwargs.pop('timeout', None), - version_id=version_id, - snapshot=self.snapshot, - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, cls=kwargs.pop('cls', None) or deserialize_blob_properties, - cpk_info=cpk_info, - **kwargs)) + **options + )) except HttpResponseError as error: process_storage_error(error) blob_props.name = self.blob_name diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index bb77494f0361..3705692a0d1d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -84,11 +84,13 @@ def _parse_url( return parsed_url, sas_token, path_snapshot + def _format_url(container_name: Union[bytes, str], scheme: str, blob_name: str, query_str: str, hostname: str) -> str: if isinstance(container_name, str): container_name = container_name.encode('UTF-8') return f"{scheme}://{hostname}/{quote(container_name)}/{quote(blob_name, safe='~/')}{query_str}" + def _encode_source_url(source_url: str) -> str: parsed_source_url = urlparse(source_url) source_scheme = parsed_source_url.scheme @@ -100,6 +102,7 @@ def _encode_source_url(source_url: str) -> str: result.append(source_query) return '?'.join(result) + def _upload_blob_options( # pylint:disable=too-many-statements data: Union[bytes, str, Iterable[AnyStr], AsyncIterable[AnyStr], IO[bytes]], blob_type: Union[str, BlobType], @@ -189,6 +192,7 @@ def _upload_blob_options( # pylint:disable=too-many-statements raise ValueError(f"Unsupported BlobType: {blob_type}") return {k: v for k, v in kwargs.items() if v is not None} + def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, Any]: metadata = kwargs.pop('metadata', None) headers = kwargs.pop('headers', {}) @@ -237,6 +241,7 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A options['modified_access_conditions'].if_none_match = '*' return options + def _download_blob_options( blob_name: str, container_name: str, @@ -326,6 +331,7 @@ def _download_blob_options( options.update({k: v for k, v in kwargs.items() if v is not None}) return options + def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwargs: Any) -> Tuple[Dict[str, Any], str]: delimiter = '\n' input_format = kwargs.pop('blob_format', None) @@ -388,6 +394,7 @@ def _quick_query_options(snapshot: Optional[str], query_expression: str, **kwarg options.update({k: v for k, v in kwargs.items() if v is not None}) return options, delimiter + def _generic_delete_blob_options(delete_snapshots: Optional[str] = None, **kwargs: Any) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -403,6 +410,7 @@ def _generic_delete_blob_options(delete_snapshots: Optional[str] = None, **kwarg options.update({k: v for k, v in kwargs.items() if v is not None}) return options + def _delete_blob_options( snapshot: Optional[str], version_id: Optional[str], @@ -417,6 +425,7 @@ def _delete_blob_options( options['blob_delete_type'] = kwargs.pop('blob_delete_type', None) return options + def _set_http_headers_options(content_settings: Optional["ContentSettings"] = None, **kwargs: Any) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -439,6 +448,7 @@ def _set_http_headers_options(content_settings: Optional["ContentSettings"] = No options.update(kwargs) return options + def _set_blob_metadata_options(metadata: Optional[Dict[str, str]] = None, **kwargs: Any): headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) @@ -462,6 +472,7 @@ def _set_blob_metadata_options(metadata: Optional[Dict[str, str]] = None, **kwar options.update(kwargs) return options + def _create_page_blob_options( size: int, content_settings: Optional["ContentSettings"] = None, @@ -523,6 +534,7 @@ def _create_page_blob_options( options.update(kwargs) return options + def _create_append_blob_options( content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, @@ -571,6 +583,7 @@ def _create_append_blob_options( options.update(kwargs) return options + def _create_snapshot_options(metadata: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]: headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) @@ -594,6 +607,7 @@ def _create_snapshot_options(metadata: Optional[Dict[str, str]] = None, **kwargs options.update(kwargs) return options + def _start_copy_from_url_options( # pylint:disable=too-many-statements source_url: str, metadata: Optional[Dict[str, str]] = None, @@ -676,6 +690,7 @@ def _start_copy_from_url_options( # pylint:disable=too-many-statements options.update(kwargs) return options + def _abort_copy_options(copy_id: Union[str, Dict[str, Any], BlobProperties], **kwargs: Any) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop('lease', None)) if isinstance(copy_id, BlobProperties): @@ -689,6 +704,7 @@ def _abort_copy_options(copy_id: Union[str, Dict[str, Any], BlobProperties], **k options.update(kwargs) return options + def _stage_block_options( block_id: str, data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], @@ -729,6 +745,7 @@ def _stage_block_options( options.update(kwargs) return options + def _stage_block_from_url_options( block_id: str, source_url: str, @@ -771,6 +788,7 @@ def _stage_block_from_url_options( options.update(kwargs) return options + def _get_block_list_result(blocks: BlockList) -> Tuple[List[BlobBlock], List[BlobBlock]]: committed = [] uncommitted = [] @@ -780,6 +798,7 @@ def _get_block_list_result(blocks: BlockList) -> Tuple[List[BlobBlock], List[Blo uncommitted = [BlobBlock._from_generated(b) for b in blocks.uncommitted_blocks] # pylint: disable=protected-access return committed, uncommitted + def _commit_block_list_options( block_list: List[BlobBlock], content_settings: Optional["ContentSettings"] = None, @@ -845,6 +864,7 @@ def _commit_block_list_options( options.update(kwargs) return options + def _set_blob_tags_options( version_id: Optional[str], tags: Optional[Dict[str, str]] = None, @@ -863,6 +883,7 @@ def _set_blob_tags_options( options.update(kwargs) return options + def _get_blob_tags_options(version_id: Optional[str], snapshot: Optional[str], **kwargs: Any) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -876,6 +897,7 @@ def _get_blob_tags_options(version_id: Optional[str], snapshot: Optional[str], * 'cls': return_headers_and_deserialized} return options + def _get_page_ranges_options( snapshot: Optional[str], offset: Optional[int] = None, @@ -909,6 +931,7 @@ def _get_page_ranges_options( options.update(kwargs) return options + def _set_sequence_number_options( sequence_number_action: str, sequence_number: Optional[str] = None, @@ -928,6 +951,7 @@ def _set_sequence_number_options( options.update(kwargs) return options + def _resize_blob_options(size: int, **kwargs: Any) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -949,6 +973,7 @@ def _resize_blob_options(size: int, **kwargs: Any) -> Dict[str, Any]: options.update(kwargs) return options + def _upload_page_options( page: bytes, offset: int, @@ -993,6 +1018,7 @@ def _upload_page_options( options.update(kwargs) return options + def _upload_pages_from_url_options( source_url: str, offset: int, @@ -1049,6 +1075,7 @@ def _upload_pages_from_url_options( options.update(kwargs) return options + def _clear_page_options( offset: int, length: int, @@ -1086,6 +1113,7 @@ def _clear_page_options( options.update(kwargs) return options + def _append_block_options( data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], length: Optional[int] = None, @@ -1134,6 +1162,7 @@ def _append_block_options( options.update(kwargs) return options + def _append_block_from_url_options( copy_source_url: str, source_offset: Optional[int] = None, @@ -1190,6 +1219,7 @@ def _append_block_from_url_options( options.update(kwargs) return options + def _seal_append_blob_options(**kwargs: Any) -> Dict[str, Any]: appendpos_condition = kwargs.pop('appendpos_condition', None) append_conditions = None @@ -1209,6 +1239,7 @@ def _seal_append_blob_options(**kwargs: Any) -> Dict[str, Any]: options.update(kwargs) return options + def _from_blob_url( blob_url: str, snapshot: Optional[Union[BlobProperties, str, Dict[str, Any]]] @@ -1254,3 +1285,25 @@ def _from_blob_url( else: path_snapshot = snapshot return (account_url, container_name, blob_name, path_snapshot) + + +def _get_blob_properties_options(version_id: Optional[str], snapshot: Optional[str], **kwargs: Any) -> Dict[str, Any]: + access_conditions = get_access_conditions(kwargs.pop('lease', None)) + mod_conditions = get_modify_conditions(kwargs) + cpk = kwargs.pop('cpk', None) + cpk_info = None + if cpk: + CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) + options = { + 'version_id': version_id, + 'lease_access_conditions': access_conditions, + 'modified_access_conditions': mod_conditions, + 'snapshot': snapshot, + 'cpk_info': cpk_info, + } + options.update({k: v for k, v in kwargs.items() if v is not None}) + return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index da835e3a1ad3..5aba53869d67 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -43,6 +43,7 @@ _download_blob_options, _format_url, _from_blob_url, + _get_blob_properties_options, _get_blob_tags_options, _get_block_list_result, _get_page_ranges_options, @@ -69,7 +70,6 @@ ) from .._encryption import StorageEncryptionMixin, _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from .._generated.aio import AzureBlobStorage -from .._generated.models import CpkInfo from .._models import BlobType, BlobBlock, BlobProperties, PageRange from .._serialize import get_access_conditions, get_api_version, get_modify_conditions, get_version_id from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper, parse_connection_str @@ -1029,7 +1029,7 @@ async def delete_blob( process_storage_error(error) @distributed_trace_async - async def undelete_blob(self, **kwargs: Any) -> None: + async def undelete_blob(self, *, timeout: Optional[int] = None, **kwargs: Any) -> None: """Restores soft-deleted blobs or snapshots. Operation will only be successful if used within the specified number of days @@ -1057,12 +1057,12 @@ async def undelete_blob(self, **kwargs: Any) -> None: :caption: Undeleting a blob. """ try: - await self._client.blob.undelete(timeout=kwargs.pop('timeout', None), **kwargs) + await self._client.blob.undelete(timeout=timeout, **kwargs) except HttpResponseError as error: process_storage_error(error) @distributed_trace_async - async def exists(self, **kwargs: Any) -> bool: + async def exists(self, *, version_id: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any) -> bool: """ Returns True if a blob exists with the defined parameters, and returns False otherwise. @@ -1079,12 +1079,13 @@ async def exists(self, **kwargs: Any) -> bool: :returns: boolean :rtype: bool """ - version_id = get_version_id(self.version_id, kwargs) try: await self._client.blob.get_properties( snapshot=self.snapshot, - version_id=version_id, - **kwargs) + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + ) return True # Encrypted with CPK except ResourceExistsError: @@ -1096,7 +1097,19 @@ async def exists(self, **kwargs: Any) -> bool: return False @distributed_trace_async - async def get_blob_properties(self, **kwargs: Any) -> BlobProperties: + async def get_blob_properties( + self, *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + version_id: Optional[str] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> BlobProperties: """Returns all user-defined metadata, standard HTTP properties, and system properties for the blob. It does not return the content of the blob. @@ -1158,29 +1171,29 @@ async def get_blob_properties(self, **kwargs: Any) -> BlobProperties: :dedent: 12 :caption: Getting the properties for a blob. """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) - version_id = get_version_id(self.version_id, kwargs) - cpk = kwargs.pop('cpk', None) - cpk_info = None - if cpk: - if self.scheme.lower() != 'https': - raise ValueError("Customer provided encryption key must be used over HTTPS.") - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + if cpk and self.scheme.lower() != 'https': + raise ValueError("Customer provided encryption key must be used over HTTPS.") + options = _get_blob_properties_options( + lease=lease, + version_id=version_id or self.version_id, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + snapshot=self.snapshot, + timeout=timeout, + **kwargs + ) try: cls_method = kwargs.pop('cls', None) if cls_method: kwargs['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) blob_props = await self._client.blob.get_properties( - timeout=kwargs.pop('timeout', None), - version_id=version_id, - snapshot=self.snapshot, - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, cls=kwargs.pop('cls', None) or deserialize_blob_properties, - cpk_info=cpk_info, - **kwargs) + **options + ) except HttpResponseError as error: process_storage_error(error) blob_props.name = self.blob_name From d44278c361b218de11ff2bbfdcb3378b0e731c75 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Fri, 28 Mar 2025 15:00:30 -0400 Subject: [PATCH 20/43] Fixed cpk in get_blob_properties --- .../azure/storage/blob/_blob_client_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 3705692a0d1d..d9b606f3ab99 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -1293,7 +1293,7 @@ def _get_blob_properties_options(version_id: Optional[str], snapshot: Optional[s cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - CpkInfo( + cpk_info = CpkInfo( encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, encryption_algorithm=cpk.algorithm From 613d33e0ea9c2019899c5cbd6da3c74a9efe72c5 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Fri, 28 Mar 2025 15:05:21 -0400 Subject: [PATCH 21/43] BlobQueryError import fix --- .../azure/storage/blob/_blob_client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 223f59aae8a1..194f2335e9e6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -62,7 +62,14 @@ from ._encryption import StorageEncryptionMixin, _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from ._generated import AzureBlobStorage from ._lease import BlobLeaseClient -from ._models import BlobBlock, BlobProperties, BlobType, PageRange, PageRangePaged +from ._models import ( + BlobBlock, + BlobProperties, + BlobQueryError, + BlobType, + PageRange, + PageRangePaged +) from ._quick_query_helper import BlobQueryReader from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper from ._shared.response_handlers import process_storage_error, return_response_headers @@ -90,7 +97,6 @@ ArrowDialect ) from ._models import ( - BlobQueryError, ContentSettings, ImmutabilityPolicy, PremiumPageBlobTier, @@ -942,7 +948,7 @@ def download_blob( def query_blob( self, query_expression: str, *, - on_error: Optional[Callable[["BlobQueryError"], None]] = None, + on_error: Optional[Callable[[BlobQueryError], None]] = None, blob_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", str]] = None, output_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", List["ArrowDialect"], str]] = None, # pylint: disable=line-too-long lease: Optional[Union[BlobLeaseClient, str]] = None, From 1206998fea6d4b232bf428253032116522ca6faa Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:11:40 -0400 Subject: [PATCH 22/43] set_blob_metadata --- .../azure/storage/blob/_blob_client.py | 26 +++++++++++++++++-- .../storage/blob/_blob_client_helpers.py | 12 ++++++--- .../storage/blob/aio/_blob_client_async.py | 26 +++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 194f2335e9e6..bb54d8916bd2 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1409,6 +1409,16 @@ def set_http_headers(self, content_settings: Optional["ContentSettings"] = None, @distributed_trace def set_blob_metadata( self, metadata: Optional[Dict[str, str]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Sets user-defined metadata for the blob as one or more name-value pairs. @@ -1467,9 +1477,21 @@ def set_blob_metadata( :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Union[str, datetime]] """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _set_blob_metadata_options(metadata=metadata, **kwargs) + options = _set_blob_metadata_options( + metadata=metadata, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Union[str, datetime]], self._client.blob.set_metadata(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index d9b606f3ab99..7149e7ae978a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -459,8 +459,11 @@ def _set_blob_metadata_options(metadata: Optional[Dict[str, str]] = None, **kwar cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'timeout': kwargs.pop('timeout', None), 'lease_access_conditions': access_conditions, @@ -468,8 +471,9 @@ def _set_blob_metadata_options(metadata: Optional[Dict[str, str]] = None, **kwar 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, 'cls': return_response_headers, - 'headers': headers} - options.update(kwargs) + 'headers': headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 5aba53869d67..eb8ca5b8fd55 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1259,6 +1259,16 @@ async def set_http_headers( @distributed_trace_async async def set_blob_metadata( self, metadata: Optional[Dict[str, str]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Sets user-defined metadata for the blob as one or more name-value pairs. @@ -1317,9 +1327,21 @@ async def set_blob_metadata( :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Union[str, datetime]] """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _set_blob_metadata_options(metadata=metadata, **kwargs) + options = _set_blob_metadata_options( + metadata=metadata, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Union[str, datetime]], await self._client.blob.set_metadata(**options)) except HttpResponseError as error: From 2fe75dcacdd30665c8d7e90b833c2c969279a4ff Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:16:40 -0400 Subject: [PATCH 23/43] set/delete_immutability_policy --- .../azure/storage/blob/_blob_client.py | 26 ++++++++++++++----- .../storage/blob/aio/_blob_client_async.py | 24 +++++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index bb54d8916bd2..621e88de8d4a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1500,6 +1500,9 @@ def set_blob_metadata( @distributed_trace def set_immutability_policy( self, immutability_policy: "ImmutabilityPolicy", + *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, str]: """The Set Immutability Policy operation sets the immutability policy on the blob. @@ -1525,15 +1528,22 @@ def set_immutability_policy( :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - - version_id = get_version_id(self.version_id, kwargs) kwargs['immutability_policy_expiry'] = immutability_policy.expiry_time kwargs['immutability_policy_mode'] = immutability_policy.policy_mode return cast(Dict[str, str], self._client.blob.set_immutability_policy( - cls=return_response_headers, version_id=version_id, **kwargs)) + cls=return_response_headers, + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + )) @distributed_trace - def delete_immutability_policy(self, **kwargs: Any) -> None: + def delete_immutability_policy( + self, *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """The Delete Immutability Policy operation deletes the immutability policy on the blob. .. versionadded:: 12.10.0 @@ -1551,9 +1561,11 @@ def delete_immutability_policy(self, **kwargs: Any) -> None: :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - - version_id = get_version_id(self.version_id, kwargs) - self._client.blob.delete_immutability_policy(version_id=version_id, **kwargs) + self._client.blob.delete_immutability_policy( + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + ) @distributed_trace def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Union[str, datetime, bool]]: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index eb8ca5b8fd55..c076d5bf7b12 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1350,6 +1350,9 @@ async def set_blob_metadata( @distributed_trace_async async def set_immutability_policy( self, immutability_policy: "ImmutabilityPolicy", + *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, str]: """The Set Immutability Policy operation sets the immutability policy on the blob. @@ -1380,10 +1383,19 @@ async def set_immutability_policy( kwargs['immutability_policy_expiry'] = immutability_policy.expiry_time kwargs['immutability_policy_mode'] = immutability_policy.policy_mode return cast(Dict[str, str], await self._client.blob.set_immutability_policy( - cls=return_response_headers,version_id=version_id, **kwargs)) + cls=return_response_headers, + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + )) @distributed_trace_async - async def delete_immutability_policy(self, **kwargs: Any) -> None: + async def delete_immutability_policy( + self, *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """The Delete Immutability Policy operation deletes the immutability policy on the blob. .. versionadded:: 12.10.0 @@ -1401,9 +1413,11 @@ async def delete_immutability_policy(self, **kwargs: Any) -> None: :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - - version_id = get_version_id(self.version_id, kwargs) - await self._client.blob.delete_immutability_policy(version_id=version_id, **kwargs) + await self._client.blob.delete_immutability_policy( + version_id=version_id or self.version_id, + timeout=timeout, + **kwargs + ) @distributed_trace_async async def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Union[str, datetime, bool]]: From cbab54e405c74231ab538f0c88014a3ad8b5aa9c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:19:14 -0400 Subject: [PATCH 24/43] set_legal_hold --- .../azure/storage/blob/_blob_client.py | 17 +++++++++++++---- .../storage/blob/aio/_blob_client_async.py | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 621e88de8d4a..8c2209d872e0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1568,7 +1568,13 @@ def delete_immutability_policy( ) @distributed_trace - def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Union[str, datetime, bool]]: + def set_legal_hold( + self, legal_hold: bool, + *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime, bool]]: """The Set Legal Hold operation sets a legal hold on the blob. .. versionadded:: 12.10.0 @@ -1588,10 +1594,13 @@ def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Union[str :returns: Key value pairs of blob tags. :rtype: Dict[str, Union[str, datetime, bool]] """ - - version_id = get_version_id(self.version_id, kwargs) return cast(Dict[str, Union[str, datetime, bool]], self._client.blob.set_legal_hold( - legal_hold, version_id=version_id, cls=return_response_headers, **kwargs)) + legal_hold, + version_id=version_id or self.version_id, + timeout=timeout, + cls=return_response_headers, + **kwargs + )) @distributed_trace def create_page_blob( diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index c076d5bf7b12..a90f74b99501 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1420,7 +1420,13 @@ async def delete_immutability_policy( ) @distributed_trace_async - async def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Union[str, datetime, bool]]: + async def set_legal_hold( + self, legal_hold: bool, + *, + version_id: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime, bool]]: """The Set Legal Hold operation sets a legal hold on the blob. .. versionadded:: 12.10.0 @@ -1440,10 +1446,13 @@ async def set_legal_hold(self, legal_hold: bool, **kwargs: Any) -> Dict[str, Uni :returns: Key value pairs of blob tags. :rtype: Dict[str, Union[str, datetime, bool]] """ - - version_id = get_version_id(self.version_id, kwargs) return cast(Dict[str, Union[str, datetime, bool]], await self._client.blob.set_legal_hold( - legal_hold, version_id=version_id, cls=return_response_headers, **kwargs)) + legal_hold, + version_id=version_id or self.version_id, + timeout=timeout, + cls=return_response_headers, + **kwargs + )) @distributed_trace_async async def create_page_blob( From 77b13ab3c3b25c45aa115c882b6754dd6a46ad46 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:28:30 -0400 Subject: [PATCH 25/43] create_page_blob --- .../azure/storage/blob/_blob_client.py | 30 +++++++++++++++++-- .../storage/blob/_blob_client_helpers.py | 12 +++++--- .../storage/blob/aio/_blob_client_async.py | 30 +++++++++++++++++-- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 8c2209d872e0..844c1543cc1a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1608,6 +1608,19 @@ def create_page_blob( content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, premium_page_blob_tier: Optional[Union[str, "PremiumPageBlobTier"]] = None, + *, + tags: Optional[Dict[str, str]] = None, + sequence_number: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a new Page Blob of the specified size. @@ -1696,14 +1709,27 @@ def create_page_blob( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _create_page_blob_options( size=size, content_settings=content_settings, metadata=metadata, premium_page_blob_tier=premium_page_blob_tier, - **kwargs) + tags=tags, + sequence_number=sequence_number, + lease=lease, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.page_blob.create(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 7149e7ae978a..5c1a4e51f3e4 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -504,8 +504,11 @@ def _create_page_blob_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) immutability_policy = kwargs.pop('immutability_policy', None) if immutability_policy: @@ -534,8 +537,9 @@ def _create_page_blob_options( 'blob_tags_string': blob_tags_string, 'cls': return_response_headers, "tier": tier, - 'headers': headers} - options.update(kwargs) + 'headers': headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index a90f74b99501..ad9f6d16c713 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1460,6 +1460,19 @@ async def create_page_blob( content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, premium_page_blob_tier: Optional[Union[str, "PremiumPageBlobTier"]] = None, + *, + tags: Optional[Dict[str, str]] = None, + sequence_number: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a new Page Blob of the specified size. @@ -1548,14 +1561,27 @@ async def create_page_blob( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _create_page_blob_options( size=size, content_settings=content_settings, metadata=metadata, premium_page_blob_tier=premium_page_blob_tier, - **kwargs) + tags=tags, + sequence_number=sequence_number, + lease=lease, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.page_blob.create(**options)) except HttpResponseError as error: From 9c3975e1f0d10543c22ba3193420d3416b427fc6 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:29:25 -0400 Subject: [PATCH 26/43] removed disable docstring should match keyword only --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 2 +- .../azure/storage/blob/aio/_blob_client_async.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 844c1543cc1a..e849ab8b75c0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only, too-many-locals +# pylint: disable=too-many-lines, too-many-locals import warnings from datetime import datetime diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index ad9f6d16c713..b203d1e8f31d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only, too-many-locals +# pylint: disable=too-many-lines, too-many-locals import warnings from datetime import datetime From cbde1e4c5673deb3952a684e331897e9bc51ab11 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:45:18 -0400 Subject: [PATCH 27/43] create_append_blob --- .../azure/storage/blob/_blob_client.py | 28 +++++++++++++++++-- .../storage/blob/_blob_client_helpers.py | 12 +++++--- .../storage/blob/aio/_blob_client_async.py | 28 +++++++++++++++++-- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index e849ab8b75c0..c1c02d74ff47 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1739,6 +1739,18 @@ def create_page_blob( def create_append_blob( self, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, + *, + tags: Optional[Dict[str, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a new Append Blob. This operation creates a new 0-length append blob. The content @@ -1818,12 +1830,24 @@ def create_append_blob( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _create_append_blob_options( content_settings=content_settings, metadata=metadata, - **kwargs) + tags=tags, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Union[str, datetime]], self._client.append_blob.create(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 5c1a4e51f3e4..11dbea651955 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -567,8 +567,11 @@ def _create_append_blob_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) immutability_policy = kwargs.pop('immutability_policy', None) if immutability_policy: @@ -587,8 +590,9 @@ def _create_append_blob_options( 'cpk_info': cpk_info, 'blob_tags_string': blob_tags_string, 'cls': return_response_headers, - 'headers': headers} - options.update(kwargs) + 'headers': headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index b203d1e8f31d..2bd9b284efb8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1591,6 +1591,18 @@ async def create_page_blob( async def create_append_blob( self, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, + *, + tags: Optional[Dict[str, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a new Append Blob. This operation creates a new 0-length append blob. The content @@ -1670,12 +1682,24 @@ async def create_append_blob( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _create_append_blob_options( content_settings=content_settings, metadata=metadata, - **kwargs) + tags=tags, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Union[str, datetime]], await self._client.append_blob.create(**options)) except HttpResponseError as error: From bef6ff3f6942c1b57468b3ac952b2d316277a12b Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 22:51:26 -0400 Subject: [PATCH 28/43] create_snapshot --- .../azure/storage/blob/_blob_client.py | 26 +++++++++++++++++-- .../storage/blob/_blob_client_helpers.py | 12 ++++++--- .../storage/blob/aio/_blob_client_async.py | 26 +++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index c1c02d74ff47..584d96df8389 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1856,6 +1856,16 @@ def create_append_blob( @distributed_trace def create_snapshot( self, metadata: Optional[Dict[str, str]] = None, + *, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a snapshot of the blob. @@ -1928,9 +1938,21 @@ def create_snapshot( :dedent: 8 :caption: Create a snapshot of the blob. """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _create_snapshot_options(metadata=metadata, **kwargs) + options = _create_snapshot_options( + metadata=metadata, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + lease=lease, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.blob.create_snapshot(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 11dbea651955..de3ed4727fe8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -605,8 +605,11 @@ def _create_snapshot_options(metadata: Optional[Dict[str, str]] = None, **kwargs cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'timeout': kwargs.pop('timeout', None), @@ -615,8 +618,9 @@ def _create_snapshot_options(metadata: Optional[Dict[str, str]] = None, **kwargs 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, 'cls': return_response_headers, - 'headers': headers} - options.update(kwargs) + 'headers': headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 2bd9b284efb8..e0cbbc7ac265 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1708,6 +1708,16 @@ async def create_append_blob( @distributed_trace_async async def create_snapshot( self, metadata: Optional[Dict[str, str]] = None, + *, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Creates a snapshot of the blob. @@ -1781,9 +1791,21 @@ async def create_snapshot( :dedent: 12 :caption: Create a snapshot of the blob. """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _create_snapshot_options(metadata=metadata, **kwargs) + options = _create_snapshot_options( + metadata=metadata, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + lease=lease, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.blob.create_snapshot(**options)) except HttpResponseError as error: From 5a27b8758b387e5f505d5ab1344c8ed409304e52 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 6 Apr 2025 23:34:01 -0400 Subject: [PATCH 29/43] Fixed cls bug for ADLS calls to get_blob_properties --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 6 +++--- .../azure/storage/blob/aio/_blob_client_async.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 584d96df8389..4061570b3327 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -1340,11 +1340,11 @@ def get_blob_properties( **kwargs ) try: - cls_method = kwargs.pop('cls', None) + cls_method = options.pop('cls', None) if cls_method: - kwargs['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) + options['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) blob_props = cast(BlobProperties, self._client.blob.get_properties( - cls=kwargs.pop('cls', None) or deserialize_blob_properties, + cls=options.pop('cls', None) or deserialize_blob_properties, **options )) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index e0cbbc7ac265..770665bf279a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -1187,11 +1187,11 @@ async def get_blob_properties( **kwargs ) try: - cls_method = kwargs.pop('cls', None) + cls_method = options.pop('cls', None) if cls_method: - kwargs['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) + options['cls'] = partial(deserialize_pipeline_response_into_cls, cls_method) blob_props = await self._client.blob.get_properties( - cls=kwargs.pop('cls', None) or deserialize_blob_properties, + cls=options.pop('cls', None) or deserialize_blob_properties, **options ) except HttpResponseError as error: From 5b781421d0b0b13a1280d56b27d9cc5865782e74 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 00:25:40 -0400 Subject: [PATCH 30/43] start_copy_from_url --- .../azure/storage/blob/_blob_client.py | 72 ++++++++++++++++--- .../storage/blob/_blob_client_helpers.py | 7 +- .../storage/blob/aio/_blob_client_async.py | 63 +++++++++++++--- 3 files changed, 119 insertions(+), 23 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 4061570b3327..b748fe029189 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -10,7 +10,7 @@ from functools import partial from typing import ( Any, AnyStr, Callable, cast, Dict, IO, Iterable, - List, Optional, overload, Tuple, Union, + List, Literal, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -95,7 +95,8 @@ DelimitedJsonDialect, QuickQueryDialect, ArrowDialect -) + ) + from ._generated.models import RehydratePriority from ._models import ( ContentSettings, ImmutabilityPolicy, @@ -1962,9 +1963,32 @@ def create_snapshot( def start_copy_from_url( self, source_url: str, metadata: Optional[Dict[str, str]] = None, - incremental_copy: bool = False, + incremental_copy: Optional[bool] = False, + *, + tags: Optional[Union[Dict[str, str], Literal["COPY"]]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + destination_lease: Optional[Union[BlobLeaseClient, str]] = None, + source_lease: Optional[Union[BlobLeaseClient, str]] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + rehydrate_priority: Optional["RehydratePriority"] = None, + seal_destination_blob: Optional[bool] = None, + requires_sync: Optional[bool] = None, + source_authorization: Optional[str] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: + """Copies a blob from the given URL. This operation returns a dictionary containing `copy_status` and `copy_id`, @@ -2092,12 +2116,6 @@ def start_copy_from_url( Specify this to perform the Copy Blob operation only if the lease ID given matches the active lease ID of the source blob. :paramtype source_lease: ~azure.storage.blob.BlobLeaseClient or str - :keyword int timeout: - Sets the server-side timeout for the operation in seconds. For more details see - https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. - This value is not tracked or validated on the client. To configure client-side network timesouts - see `here `__. :keyword ~azure.storage.blob.PremiumPageBlobTier premium_page_blob_tier: A page blob tier value to set the blob to. The tier correlates to the size of the blob and number of allowed IOPS. This is only applicable to page blobs on @@ -2129,6 +2147,12 @@ def start_copy_from_url( .. versionadded:: 12.10.0 + :keyword int timeout: + Sets the server-side timeout for the operation in seconds. For more details see + https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. + This value is not tracked or validated on the client. To configure client-side network timesouts + see `here `__. :returns: A dictionary of copy properties (etag, last_modified, copy_id, copy_status). :rtype: dict[str, Union[str, ~datetime.datetime]] @@ -2145,7 +2169,29 @@ def start_copy_from_url( source_url=source_url, metadata=metadata, incremental_copy=incremental_copy, - **kwargs) + tags=tags, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + destination_lease=destination_lease, + source_lease=source_lease, + premium_page_blob_tier=premium_page_blob_tier, + standard_blob_tier=standard_blob_tier, + rehydrate_priority=rehydrate_priority, + seal_destination_blob=seal_destination_blob, + requires_sync=requires_sync, + source_authorization=source_authorization, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: if incremental_copy: return cast(Dict[str, Union[str, datetime]], self._client.page_blob.copy_incremental(**options)) @@ -2185,7 +2231,11 @@ def abort_copy( process_storage_error(error) @distributed_trace - def acquire_lease(self, lease_duration: int =-1, lease_id: Optional[str] = None, **kwargs: Any) -> BlobLeaseClient: + def acquire_lease( + self, lease_duration: int = -1, + lease_id: Optional[str] = None, + **kwargs: Any + ) -> BlobLeaseClient: """Requests a new lease. If the blob does not have an active lease, the Blob diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index de3ed4727fe8..4ba539febc5f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -633,7 +633,7 @@ def _start_copy_from_url_options( # pylint:disable=too-many-statements source_url = _encode_source_url(source_url=source_url) headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) - if 'source_lease' in kwargs: + if kwargs.get('source_lease') is not None: source_lease = kwargs.pop('source_lease') try: headers['x-ms-source-lease-id'] = source_lease.id @@ -703,7 +703,7 @@ def _start_copy_from_url_options( # pylint:disable=too-many-statements options['source_modified_access_conditions'] = source_mod_conditions options['lease_access_conditions'] = dest_access_conditions options['tier'] = tier.value if tier else None - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -716,7 +716,8 @@ def _abort_copy_options(copy_id: Union[str, Dict[str, Any], BlobProperties], **k options = { 'copy_id': copy_id, 'lease_access_conditions': access_conditions, - 'timeout': kwargs.pop('timeout', None)} + 'timeout': kwargs.pop('timeout', None) + } options.update(kwargs) return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 770665bf279a..96aefa90145d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -10,7 +10,7 @@ from functools import partial from typing import ( Any, AnyStr, AsyncIterable, Awaitable, Callable, cast, Dict, - IO, Iterable, List, Optional, overload, Tuple, Union, + IO, Iterable, List, Literal, Optional, overload, Tuple, Union, TYPE_CHECKING ) from typing_extensions import Self @@ -83,6 +83,7 @@ from azure.core.pipeline.policies import AsyncHTTPPolicy from azure.storage.blob import CustomerProvidedEncryptionKey from azure.storage.blob.aio import ContainerClient + from .._generated.models import RehydratePriority from .._models import ( ContentSettings, ImmutabilityPolicy, @@ -1816,6 +1817,28 @@ async def start_copy_from_url( self, source_url: str, metadata: Optional[Dict[str, str]] = None, incremental_copy: bool = False, + *, + tags: Optional[Union[Dict[str, str], Literal["COPY"]]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + destination_lease: Optional[Union[BlobLeaseClient, str]] = None, + source_lease: Optional[Union[BlobLeaseClient, str]] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + rehydrate_priority: Optional["RehydratePriority"] = None, + seal_destination_blob: Optional[bool] = None, + requires_sync: Optional[bool] = None, + source_authorization: Optional[str] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Copies a blob from the given URL. @@ -1951,12 +1974,6 @@ async def start_copy_from_url( Specify this to perform the Copy Blob operation only if the lease ID given matches the active lease ID of the source blob. :paramtype source_lease: ~azure.storage.blob.aio.BlobLeaseClient or str - :keyword int timeout: - Sets the server-side timeout for the operation in seconds. For more details see - https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. - This value is not tracked or validated on the client. To configure client-side network timesouts - see `here `__. :keyword ~azure.storage.blob.PremiumPageBlobTier premium_page_blob_tier: A page blob tier value to set the blob to. The tier correlates to the size of the blob and number of allowed IOPS. This is only applicable to page blobs on @@ -1988,6 +2005,12 @@ async def start_copy_from_url( .. versionadded:: 12.10.0 + :keyword int timeout: + Sets the server-side timeout for the operation in seconds. For more details see + https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. + This value is not tracked or validated on the client. To configure client-side network timesouts + see `here `__. :returns: A dictionary of copy properties (etag, last_modified, copy_id, copy_status). :rtype: dict[str, Union[str, ~datetime.datetime]] @@ -2004,7 +2027,29 @@ async def start_copy_from_url( source_url=source_url, metadata=metadata, incremental_copy=incremental_copy, - **kwargs) + tags=tags, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + destination_lease=destination_lease, + source_lease=source_lease, + premium_page_blob_tier=premium_page_blob_tier, + standard_blob_tier=standard_blob_tier, + rehydrate_priority=rehydrate_priority, + seal_destination_blob=seal_destination_blob, + requires_sync=requires_sync, + source_authorization=source_authorization, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: if incremental_copy: return cast(Dict[str, Union[str, datetime]], await self._client.page_blob.copy_incremental(**options)) @@ -2045,7 +2090,7 @@ async def abort_copy( @distributed_trace_async async def acquire_lease( - self, lease_duration: int =-1, + self, lease_duration: int = -1, lease_id: Optional[str] = None, **kwargs: Any ) -> BlobLeaseClient: From ba31f52acf6520fe28cff3b607c50fa44a2f79fb Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 00:30:43 -0400 Subject: [PATCH 31/43] acquire_lease --- .../azure/storage/blob/_blob_client.py | 18 +++++++++++++++++- .../storage/blob/aio/_blob_client_async.py | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index b748fe029189..dc2cc4459b49 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -2234,6 +2234,13 @@ def abort_copy( def acquire_lease( self, lease_duration: int = -1, lease_id: Optional[str] = None, + *, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> BlobLeaseClient: """Requests a new lease. @@ -2292,7 +2299,16 @@ def acquire_lease( :caption: Acquiring a lease on a blob. """ lease = BlobLeaseClient(self, lease_id=lease_id) - lease.acquire(lease_duration=lease_duration, **kwargs) + lease.acquire( + lease_duration=lease_duration, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) return lease @distributed_trace diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 96aefa90145d..2e3d1ce967a9 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -2092,6 +2092,13 @@ async def abort_copy( async def acquire_lease( self, lease_duration: int = -1, lease_id: Optional[str] = None, + *, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> BlobLeaseClient: """Requests a new lease. @@ -2150,7 +2157,16 @@ async def acquire_lease( :caption: Acquiring a lease on a blob. """ lease = BlobLeaseClient(self, lease_id=lease_id) - await lease.acquire(lease_duration=lease_duration, **kwargs) + await lease.acquire( + lease_duration=lease_duration, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) return lease @distributed_trace_async From 7f0d109c56698bc0fd56c4da14108d5f5588eda1 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 00:54:30 -0400 Subject: [PATCH 32/43] set_standard_blob_tier --- .../azure/storage/blob/_blob_client.py | 46 +++++++++++-------- .../storage/blob/_blob_client_helpers.py | 20 ++++++++ .../storage/blob/aio/_blob_client_async.py | 41 ++++++++++------- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index dc2cc4459b49..1d3d41029bc9 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -44,6 +44,7 @@ _set_blob_tags_options, _set_http_headers_options, _set_sequence_number_options, + _set_standard_blob_tier_options, _stage_block_from_url_options, _stage_block_options, _start_copy_from_url_options, @@ -1963,7 +1964,7 @@ def create_snapshot( def start_copy_from_url( self, source_url: str, metadata: Optional[Dict[str, str]] = None, - incremental_copy: Optional[bool] = False, + incremental_copy: bool = False, *, tags: Optional[Union[Dict[str, str], Literal["COPY"]]] = None, immutability_policy: Optional["ImmutabilityPolicy"] = None, @@ -2312,7 +2313,16 @@ def acquire_lease( return lease @distributed_trace - def set_standard_blob_tier(self, standard_blob_tier: Union[str, "StandardBlobTier"], **kwargs: Any) -> None: + def set_standard_blob_tier( + self, standard_blob_tier: Union[str, "StandardBlobTier"], + *, + rehydrate_priority: Optional["RehydratePriority"] = None, + version_id: Optional[str] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union["BlobLeaseClient", str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """This operation sets the tier on a block blob. A block blob's tier determines Hot/Cool/Archive storage type. @@ -2341,34 +2351,30 @@ def set_standard_blob_tier(self, standard_blob_tier: Union[str, "StandardBlobTie .. versionadded:: 12.4.0 + :keyword lease: + Required if the blob has an active lease. Value can be a BlobLeaseClient object + or the lease ID as a string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword lease: - Required if the blob has an active lease. Value can be a BlobLeaseClient object - or the lease ID as a string. :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str :rtype: None """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) - version_id = get_version_id(self.version_id, kwargs) - if standard_blob_tier is None: - raise ValueError("A StandardBlobTier must be specified") - if self.snapshot and kwargs.get('version_id'): - raise ValueError("Snapshot and version_id cannot be set at the same time") + options = _set_standard_blob_tier_options( + version_id, + self.snapshot, + standard_blob_tier=standard_blob_tier, + timeout=timeout, + lease=lease, + rehydrate_priority=rehydrate_priority, + if_tags_match_condition=if_tags_match_condition, + **kwargs + ) try: - self._client.blob.set_tier( - tier=standard_blob_tier, - snapshot=self.snapshot, - timeout=kwargs.pop('timeout', None), - modified_access_conditions=mod_conditions, - lease_access_conditions=access_conditions, - version_id=version_id, - **kwargs) + self._client.blob.set_tier(**options) except HttpResponseError as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 4ba539febc5f..457cb101bb2d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -1324,3 +1324,23 @@ def _get_blob_properties_options(version_id: Optional[str], snapshot: Optional[s } options.update({k: v for k, v in kwargs.items() if v is not None}) return options + + +def _set_standard_blob_tier_options(version_id: Optional[str], snapshot: Optional[str], **kwargs): + access_conditions = get_access_conditions(kwargs.pop("lease", None)) + mod_conditions = get_modify_conditions(kwargs) + standard_blob_tier = kwargs.pop("standard_blob_tier", None) + if standard_blob_tier is None: + raise ValueError("A StandardBlobTier must be specified") + if snapshot and version_id: + raise ValueError("Snapshot and version_id cannot be set at the same time") + options = { + 'version_id': version_id, + 'tier': standard_blob_tier, + 'timeout': kwargs.pop('timeout', None), + 'lease_access_conditions': access_conditions, + 'modified_access_conditions': mod_conditions, + 'snapshot': snapshot, + } + options.update({k: v for k, v in kwargs.items() if v is not None}) + return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 2e3d1ce967a9..431045223ac7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -54,6 +54,7 @@ _set_blob_tags_options, _set_http_headers_options, _set_sequence_number_options, + _set_standard_blob_tier_options, _stage_block_from_url_options, _stage_block_options, _start_copy_from_url_options, @@ -2170,7 +2171,16 @@ async def acquire_lease( return lease @distributed_trace_async - async def set_standard_blob_tier(self, standard_blob_tier: Union[str, "StandardBlobTier"], **kwargs: Any) -> None: + async def set_standard_blob_tier( + self, standard_blob_tier: Union[str, "StandardBlobTier"], + *, + rehydrate_priority: Optional["RehydratePriority"] = None, + version_id: Optional[str] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union["BlobLeaseClient", str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """This operation sets the tier on a block blob. A block blob's tier determines Hot/Cool/Archive storage type. @@ -2192,31 +2202,30 @@ async def set_standard_blob_tier(self, standard_blob_tier: Union[str, "StandardB .. versionadded:: 12.4.0 + :keyword lease: + Required if the blob has an active lease. Value can be a BlobLeaseClient object + or the lease ID as a string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword lease: - Required if the blob has an active lease. Value can be a BlobLeaseClient object - or the lease ID as a string. :paramtype lease: ~azure.storage.blob.aio.BlobLeaseClient or str :rtype: None """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) - version_id = get_version_id(self.version_id, kwargs) - if standard_blob_tier is None: - raise ValueError("A StandardBlobTier must be specified") + options = _set_standard_blob_tier_options( + version_id, + self.snapshot, + standard_blob_tier=standard_blob_tier, + timeout=timeout, + lease=lease, + rehydrate_priority=rehydrate_priority, + if_tags_match_condition=if_tags_match_condition, + **kwargs + ) try: - await self._client.blob.set_tier( - tier=standard_blob_tier, - timeout=kwargs.pop('timeout', None), - modified_access_conditions=mod_conditions, - lease_access_conditions=access_conditions, - version_id=version_id, - **kwargs) + await self._client.blob.set_tier(**options) except HttpResponseError as error: process_storage_error(error) From b16ee4a330dc478bd29e9fb50f8d8faa3e256662 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 01:15:03 -0400 Subject: [PATCH 33/43] stage_block --- .../azure/storage/blob/_blob_client.py | 22 ++++++++-- .../storage/blob/_blob_client_helpers.py | 42 ++++++++++++------- .../storage/blob/aio/_blob_client_async.py | 22 ++++++++-- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 1d3d41029bc9..c4b512b8f193 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -21,6 +21,7 @@ from azure.core.tracing.decorator import distributed_trace from ._blob_client_helpers import ( _abort_copy_options, + _acquire_lease_options, _append_block_from_url_options, _append_block_options, _clear_page_options, @@ -2300,7 +2301,7 @@ def acquire_lease( :caption: Acquiring a lease on a blob. """ lease = BlobLeaseClient(self, lease_id=lease_id) - lease.acquire( + options = _acquire_lease_options( lease_duration=lease_duration, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, @@ -2310,6 +2311,7 @@ def acquire_lease( timeout=timeout, **kwargs ) + lease.acquire(**options) return lease @distributed_trace @@ -2383,6 +2385,13 @@ def stage_block( self, block_id: str, data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], length: Optional[int] = None, + *, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + encoding: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new block to be committed as part of a blob. @@ -2432,13 +2441,20 @@ def stage_block( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _stage_block_options( block_id=block_id, data=data, length=length, - **kwargs) + validate_content=validate_content, + lease=lease, + encoding=encoding, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.block_blob.stage_block(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 457cb101bb2d..041547172c3c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -114,7 +114,7 @@ def _upload_blob_options( # pylint:disable=too-many-statements client: "AzureBlobStorage", **kwargs: Any ) -> Dict[str, Any]: - encoding = kwargs.pop('encoding', 'UTF-8') + encoding = kwargs.pop('encoding') or 'UTF-8' if isinstance(data, str): data = data.encode(encoding) if length is None: @@ -134,15 +134,18 @@ def _upload_blob_options( # pylint:disable=too-many-statements else: raise TypeError(f"Unsupported data type: {type(data)}") - validate_content = kwargs.pop('validate_content', False) - content_settings = kwargs.pop('content_settings', None) - overwrite = kwargs.pop('overwrite', False) - max_concurrency = kwargs.pop('max_concurrency', 1) - cpk = kwargs.pop('cpk', None) + validate_content = kwargs.pop('validate_content') or False + content_settings = kwargs.pop('content_settings') or None + overwrite = kwargs.pop('overwrite') or False + max_concurrency = kwargs.pop('max_concurrency') or 1 + cpk = kwargs.pop('cpk') or None cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) kwargs['cpk_info'] = cpk_info headers = kwargs.pop('headers', {}) @@ -730,7 +733,7 @@ def _stage_block_options( ) -> Dict[str, Any]: block_id = encode_base64(str(block_id)) if isinstance(data, str): - data = data.encode(kwargs.pop('encoding', 'UTF-8')) # type: ignore + data = data.encode(kwargs.pop('encoding') or 'UTF-8') # type: ignore access_conditions = get_access_conditions(kwargs.pop('lease', None)) if length is None: length = get_length(data) @@ -739,13 +742,16 @@ def _stage_block_options( if isinstance(data, bytes): data = data[:length] - validate_content = kwargs.pop('validate_content', False) + validate_content = kwargs.pop('validate_content') or False cpk_scope_info = get_cpk_scope_info(kwargs) cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'block_id': block_id, @@ -759,7 +765,7 @@ def _stage_block_options( 'cpk_info': cpk_info, 'cls': return_response_headers, } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1326,7 +1332,11 @@ def _get_blob_properties_options(version_id: Optional[str], snapshot: Optional[s return options -def _set_standard_blob_tier_options(version_id: Optional[str], snapshot: Optional[str], **kwargs): +def _set_standard_blob_tier_options( + version_id: Optional[str], + snapshot: Optional[str], + **kwargs: Any +) -> Dict[str, Any]: access_conditions = get_access_conditions(kwargs.pop("lease", None)) mod_conditions = get_modify_conditions(kwargs) standard_blob_tier = kwargs.pop("standard_blob_tier", None) @@ -1344,3 +1354,7 @@ def _set_standard_blob_tier_options(version_id: Optional[str], snapshot: Optiona } options.update({k: v for k, v in kwargs.items() if v is not None}) return options + + +def _acquire_lease_options(**kwargs: Any): + return {k: v for k, v in kwargs.items() if v is not None} diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 431045223ac7..98043d1660b1 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -32,6 +32,7 @@ from .._blob_client import StorageAccountHostsMixin from .._blob_client_helpers import ( _abort_copy_options, + _acquire_lease_options, _append_block_from_url_options, _append_block_options, _clear_page_options, @@ -2158,7 +2159,7 @@ async def acquire_lease( :caption: Acquiring a lease on a blob. """ lease = BlobLeaseClient(self, lease_id=lease_id) - await lease.acquire( + options = _acquire_lease_options( lease_duration=lease_duration, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, @@ -2168,6 +2169,7 @@ async def acquire_lease( timeout=timeout, **kwargs ) + await lease.acquire(**options) return lease @distributed_trace_async @@ -2234,6 +2236,13 @@ async def stage_block( self, block_id: str, data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], length: Optional[int] = None, + *, + validate_content: Optional[bool] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + encoding: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new block to be committed as part of a blob. @@ -2283,13 +2292,20 @@ async def stage_block( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _stage_block_options( block_id=block_id, data=data, length=length, - **kwargs) + validate_content=validate_content, + lease=lease, + encoding=encoding, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.block_blob.stage_block(**options)) except HttpResponseError as error: From e127faffb2aff215912601c703fdc30c515b099d Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 01:22:01 -0400 Subject: [PATCH 34/43] converted kwargs.pop('kwarg', default_value) into kwargs.pop('kwarg') or default_value when default_value != None --- .../azure/storage/blob/_blob_client_helpers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 041547172c3c..bc2ce859ff85 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -226,7 +226,7 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A options = { 'copy_source_authorization': source_authorization, 'content_length': 0, - 'copy_source_blob_properties': kwargs.pop('include_source_blob_properties', True), + 'copy_source_blob_properties': kwargs.pop('include_source_blob_properties') or True, 'source_content_md5': kwargs.pop('source_content_md5', None), 'copy_source': source_url, 'modified_access_conditions': get_modify_conditions(kwargs), @@ -288,7 +288,7 @@ def _download_blob_options( raise ValueError("Offset must be provided if length is provided.") length = offset + length - 1 # Service actually uses an end-range inclusive index - validate_content = kwargs.pop('validate_content', False) + validate_content = kwargs.pop('validate_content') or False access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -325,7 +325,7 @@ def _download_blob_options( 'modified_access_conditions': mod_conditions, 'cpk_info': cpk_info, 'download_cls': kwargs.pop('cls', None) or deserialize_blob_stream, - 'max_concurrency': kwargs.pop('max_concurrency', 1), + 'max_concurrency': kwargs.pop('max_concurrency') or 1, 'encoding': encoding, 'timeout': kwargs.pop('timeout', None), 'name': blob_name, @@ -854,7 +854,7 @@ def _commit_block_list_options( blob_content_disposition=content_settings.content_disposition ) - validate_content = kwargs.pop('validate_content', False) + validate_content = kwargs.pop('validate_content') or False cpk_scope_info = get_cpk_scope_info(kwargs) cpk = kwargs.pop('cpk', None) cpk_info = None @@ -1004,13 +1004,13 @@ def _upload_page_options( **kwargs: Any ) -> Dict[str, Any]: if isinstance(page, str): - page = page.encode(kwargs.pop('encoding', 'UTF-8')) + page = page.encode(kwargs.pop('encoding') or 'UTF-8') if offset is None or offset % 512 != 0: raise ValueError("offset must be an integer that aligns with 512 page size") if length is None or length % 512 != 0: raise ValueError("length must be an integer that aligns with 512 page size") end_range = offset + length - 1 # Reformat to an inclusive range index - content_range = f'bytes={offset}-{end_range}' # type: ignore + content_range = f'bytes={offset}-{end_range}' # type: ignore access_conditions = get_access_conditions(kwargs.pop('lease', None)) seq_conditions = SequenceNumberAccessConditions( if_sequence_number_less_than_or_equal_to=kwargs.pop('if_sequence_number_lte', None), @@ -1019,7 +1019,7 @@ def _upload_page_options( ) mod_conditions = get_modify_conditions(kwargs) cpk_scope_info = get_cpk_scope_info(kwargs) - validate_content = kwargs.pop('validate_content', False) + validate_content = kwargs.pop('validate_content') or False cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: @@ -1143,7 +1143,7 @@ def _append_block_options( **kwargs: Any ) -> Dict[str, Any]: if isinstance(data, str): - data = data.encode(kwargs.pop('encoding', 'UTF-8')) + data = data.encode(kwargs.pop('encoding') or 'UTF-8') if length is None: length = get_length(data) if length is None: @@ -1155,7 +1155,7 @@ def _append_block_options( appendpos_condition = kwargs.pop('appendpos_condition', None) maxsize_condition = kwargs.pop('maxsize_condition', None) - validate_content = kwargs.pop('validate_content', False) + validate_content = kwargs.pop('validate_content') or False append_conditions = None if maxsize_condition or appendpos_condition is not None: append_conditions = AppendPositionAccessConditions( From 52f0e3e7cb6ea935a79208f550f01ebc49dedc25 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 7 Apr 2025 01:55:46 -0400 Subject: [PATCH 35/43] Added implicit kwargs to from_* APIs for parity to the constructor --- .../azure/storage/blob/_blob_client.py | 36 +++++++++++++++++++ .../storage/blob/aio/_blob_client_async.py | 36 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index c4b512b8f193..edbc687fb7e7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -251,8 +251,17 @@ def from_blob_url( credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long snapshot: Optional[Union[str, Dict[str, Any]]] = None, *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, + use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> Self: """Create BlobClient from a blob url. This doesn't support customized blob url with '/' in blob name. @@ -297,6 +306,15 @@ def from_blob_url( credential=credential, version_id=version_id, audience=audience, + api_version=api_version, + secondary_hostname=secondary_hostname, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, **kwargs ) @@ -308,8 +326,17 @@ def from_connection_string( snapshot: Optional[Union[str, Dict[str, Any]]] = None, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, + use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> Self: """Create BlobClient from a Connection String. @@ -365,6 +392,15 @@ def from_connection_string( credential=credential, version_id=version_id, audience=audience, + api_version=api_version, + secondary_hostname=secondary_hostname, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, **kwargs ) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 98043d1660b1..ee55f6088448 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -235,8 +235,17 @@ def from_blob_url( credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long snapshot: Optional[Union[str, Dict[str, Any]]] = None, *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, + use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> Self: """Create BlobClient from a blob url. This doesn't support customized blob url with '/' in blob name. @@ -277,6 +286,15 @@ def from_blob_url( credential=credential, version_id=version_id, audience=audience, + api_version=api_version, + secondary_hostname=secondary_hostname, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, **kwargs ) @@ -288,8 +306,17 @@ def from_connection_string( snapshot: Optional[Union[str, Dict[str, Any]]] = None, credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long *, + api_version: Optional[str] = None, + secondary_hostname: Optional[str] = None, version_id: Optional[str] = None, audience: Optional[str] = None, + max_block_size: int = 4 * 1024 * 1024, + max_page_size: int = 4 * 1024 * 1024, + max_chunk_get_size: int = 4 * 1024 * 1024, + max_single_put_size: int = 64 * 1024 * 1024, + max_single_get_size: int = 32 * 1024 * 1024, + min_large_block_upload_threshold: int = 4 * 1024 * 1024 + 1, + use_byte_buffer: Optional[bool] = None, **kwargs: Any ) -> Self: """Create BlobClient from a Connection String. @@ -341,6 +368,15 @@ def from_connection_string( credential=credential, version_id=version_id, audience=audience, + api_version=api_version, + secondary_hostname=secondary_hostname, + max_block_size=max_block_size, + max_page_size=max_page_size, + max_chunk_get_size=max_chunk_get_size, + max_single_put_size=max_single_put_size, + max_single_get_size=max_single_get_size, + min_large_block_upload_threshold=min_large_block_upload_threshold, + use_byte_buffer=use_byte_buffer, **kwargs ) From fe5e747ffd3abb4273d7f635f8a282d1331e7f07 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 9 Apr 2025 11:50:08 -0400 Subject: [PATCH 36/43] options update only non-None kwargs --- .../storage/blob/_blob_client_helpers.py | 124 +++++++++++------- 1 file changed, 80 insertions(+), 44 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index bc2ce859ff85..ab86e04fbd99 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -447,8 +447,9 @@ def _set_http_headers_options(content_settings: Optional["ContentSettings"] = No 'blob_http_headers': blob_headers, 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -721,7 +722,7 @@ def _abort_copy_options(copy_id: Union[str, Dict[str, Any], BlobProperties], **k 'lease_access_conditions': access_conditions, 'timeout': kwargs.pop('timeout', None) } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -793,8 +794,11 @@ def _stage_block_from_url_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'copy_source_authorization': source_authorization, 'block_id': block_id, @@ -808,7 +812,7 @@ def _stage_block_from_url_options( 'cpk_info': cpk_info, 'cls': return_response_headers, } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -859,8 +863,11 @@ def _commit_block_list_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) immutability_policy = kwargs.pop('immutability_policy', None) if immutability_policy: @@ -884,7 +891,7 @@ def _commit_block_list_options( 'blob_tags_string': blob_tags_string, 'headers': headers } - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -892,7 +899,7 @@ def _set_blob_tags_options( version_id: Optional[str], tags: Optional[Dict[str, str]] = None, **kwargs: Any -)-> Dict[str, Any]: +) -> Dict[str, Any]: serialized_tags = serialize_blob_tags(tags) access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_modify_conditions(kwargs) @@ -902,8 +909,9 @@ def _set_blob_tags_options( 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, 'version_id': version_id, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -917,7 +925,8 @@ def _get_blob_tags_options(version_id: Optional[str], snapshot: Optional[str], * 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, 'timeout': kwargs.pop('timeout', None), - 'cls': return_headers_and_deserialized} + 'cls': return_headers_and_deserialized + } return options @@ -942,7 +951,8 @@ def _get_page_ranges_options( 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, 'timeout': kwargs.pop('timeout', None), - 'range': page_range} + 'range': page_range + } if previous_snapshot_diff: try: options['prevsnapshot'] = previous_snapshot_diff.snapshot # type: ignore @@ -951,7 +961,7 @@ def _get_page_ranges_options( options['prevsnapshot'] = previous_snapshot_diff['snapshot'] # type: ignore except TypeError: options['prevsnapshot'] = previous_snapshot_diff - options.update(kwargs) + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -970,8 +980,9 @@ def _set_sequence_number_options( 'blob_sequence_number': sequence_number, 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -984,16 +995,20 @@ def _resize_blob_options(size: int, **kwargs: Any) -> Dict[str, Any]: cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'blob_content_length': size, 'timeout': kwargs.pop('timeout', None), 'lease_access_conditions': access_conditions, 'modified_access_conditions': mod_conditions, 'cpk_info': cpk_info, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1023,8 +1038,11 @@ def _upload_page_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'body': page[:length], 'content_length': length, @@ -1037,8 +1055,9 @@ def _upload_page_options( 'validate_content': validate_content, 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1077,8 +1096,11 @@ def _upload_pages_from_url_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'copy_source_authorization': source_authorization, @@ -1094,8 +1116,9 @@ def _upload_pages_from_url_options( 'source_modified_access_conditions': source_mod_conditions, 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1121,8 +1144,11 @@ def _clear_page_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'content_length': 0, @@ -1132,8 +1158,9 @@ def _clear_page_options( 'sequence_number_access_conditions': seq_conditions, 'modified_access_conditions': mod_conditions, 'cpk_info': cpk_info, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1168,8 +1195,11 @@ def _append_block_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'body': data, 'content_length': length, @@ -1181,8 +1211,9 @@ def _append_block_options( 'validate_content': validate_content, 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1221,8 +1252,11 @@ def _append_block_from_url_options( cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: - cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cpk.algorithm) + cpk_info = CpkInfo( + encryption_key=cpk.key_value, + encryption_key_sha256=cpk.key_hash, + encryption_algorithm=cpk.algorithm + ) options = { 'copy_source_authorization': source_authorization, @@ -1238,8 +1272,9 @@ def _append_block_from_url_options( 'cpk_scope_info': cpk_scope_info, 'cpk_info': cpk_info, 'cls': return_response_headers, - 'timeout': kwargs.pop('timeout', None)} - options.update(kwargs) + 'timeout': kwargs.pop('timeout', None) + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1258,8 +1293,9 @@ def _seal_append_blob_options(**kwargs: Any) -> Dict[str, Any]: 'lease_access_conditions': access_conditions, 'append_position_access_conditions': append_conditions, 'modified_access_conditions': mod_conditions, - 'cls': return_response_headers} - options.update(kwargs) + 'cls': return_response_headers + } + options.update({k: v for k, v in kwargs.items() if v is not None}) return options @@ -1307,7 +1343,7 @@ def _from_blob_url( path_snapshot = snapshot['snapshot'] else: path_snapshot = snapshot - return (account_url, container_name, blob_name, path_snapshot) + return account_url, container_name, blob_name, path_snapshot def _get_blob_properties_options(version_id: Optional[str], snapshot: Optional[str], **kwargs: Any) -> Dict[str, Any]: From e115a8b67f51aa159766dec23560bc962155ffa4 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 9 Apr 2025 15:16:14 -0400 Subject: [PATCH 37/43] Rest of the APIs --- .../azure/storage/blob/_blob_client.py | 467 +++++++++++++++--- .../storage/blob/_blob_client_helpers.py | 15 + .../storage/blob/aio/_blob_client_async.py | 463 ++++++++++++++--- 3 files changed, 829 insertions(+), 116 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index edbc687fb7e7..1b96bce75f74 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -35,6 +35,7 @@ _from_blob_url, _get_blob_properties_options, _get_blob_tags_options, + _get_block_list_options, _get_block_list_result, _get_page_ranges_options, _parse_url, @@ -2503,6 +2504,12 @@ def stage_block_from_url( source_offset: Optional[int] = None, source_length: Optional[int] = None, source_content_md5: Optional[Union[bytes, bytearray]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new block to be committed as part of a blob where @@ -2536,19 +2543,19 @@ def stage_block_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Blob property dict. :rtype: Dict[str, Any] """ - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _stage_block_from_url_options( block_id=block_id, @@ -2556,7 +2563,13 @@ def stage_block_from_url( source_offset=source_offset, source_length=source_length, source_content_md5=source_content_md5, - **kwargs) + lease=lease, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.block_blob.stage_block_from_url(**options)) except HttpResponseError as error: @@ -2565,6 +2578,10 @@ def stage_block_from_url( @distributed_trace def get_block_list( self, block_list_type: str = "committed", + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[BlobBlock], List[BlobBlock]]: """The Get Block List operation retrieves the list of blocks that have @@ -2592,16 +2609,16 @@ def get_block_list( :returns: A tuple of two lists - committed and uncommitted blocks :rtype: Tuple[List[BlobBlock], List[BlobBlock]] """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) + options = _get_block_list_options( + block_list_type=block_list_type, + snapshot=self.snapshot, + lease=lease, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: - blocks = self._client.block_blob.get_block_list( - list_type=block_list_type, - snapshot=self.snapshot, - timeout=kwargs.pop('timeout', None), - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, - **kwargs) + blocks = self._client.block_blob.get_block_list(**options) except HttpResponseError as error: process_storage_error(error) return _get_block_list_result(blocks) @@ -2611,6 +2628,21 @@ def commit_block_list( self, block_list: List[BlobBlock], content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, + *, + tags: Optional[Dict[str, str]] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + validate_content: Optional[bool] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """The Commit Block List operation writes a blob by specifying the list of @@ -2706,20 +2738,41 @@ def commit_block_list( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _commit_block_list_options( block_list=block_list, content_settings=content_settings, metadata=metadata, - **kwargs) + tags=tags, + lease=lease, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + validate_content=validate_content, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + standard_blob_tier=standard_blob_tier, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.block_blob.commit_block_list(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def set_premium_page_blob_tier(self, premium_page_blob_tier: "PremiumPageBlobTier", **kwargs: Any) -> None: + def set_premium_page_blob_tier( + self, premium_page_blob_tier: "PremiumPageBlobTier", + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """Sets the page blob tiers on the blob. This API is only supported for page blobs on premium accounts. :param premium_page_blob_tier: @@ -2733,34 +2786,44 @@ def set_premium_page_blob_tier(self, premium_page_blob_tier: "PremiumPageBlobTie .. versionadded:: 12.4.0 + :keyword lease: + Required if the blob has an active lease. Value can be a BlobLeaseClient object + or the lease ID as a string. + :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword lease: - Required if the blob has an active lease. Value can be a BlobLeaseClient object - or the lease ID as a string. - :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str :rtype: None """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(lease) mod_conditions = get_modify_conditions(kwargs) if premium_page_blob_tier is None: raise ValueError("A PremiumPageBlobTier must be specified") try: self._client.blob.set_tier( tier=premium_page_blob_tier, - timeout=kwargs.pop('timeout', None), + timeout=timeout, lease_access_conditions=access_conditions, modified_access_conditions=mod_conditions, - **kwargs) + **kwargs + ) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]: + def set_blob_tags( + self, tags: Optional[Dict[str, str]] = None, + *, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Any]: """The Set Tags operation enables users to set tags on a blob or specific blob version, but not snapshot. Each call to this operation replaces all existing tags attached to the blob. To remove all tags from the blob, call this operation with no tags set. @@ -2801,15 +2864,29 @@ def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: Any) -> :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Any] """ - version_id = get_version_id(self.version_id, kwargs) - options = _set_blob_tags_options(version_id=version_id, tags=tags, **kwargs) + options = _set_blob_tags_options( + version_id=version_id or self.version_id, + tags=tags, + validate_content=validate_content, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.blob.set_tags(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def get_blob_tags(self, **kwargs: Any) -> Dict[str, str]: + def get_blob_tags( + self, *, + version_id: Optional[str] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, str]: """The Get Tags operation enables users to get tags on a blob or specific blob version, or snapshot. .. versionadded:: 12.4.0 @@ -2834,8 +2911,14 @@ def get_blob_tags(self, **kwargs: Any) -> Dict[str, str]: :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - version_id = get_version_id(self.version_id, kwargs) - options = _get_blob_tags_options(version_id=version_id, snapshot=self.snapshot, **kwargs) + options = _get_blob_tags_options( + version_id=version_id or self.version_id, + snapshot=self.snapshot, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: _, tags = self._client.blob.get_tags(**options) return cast(Dict[str, str], parse_tags(tags)) @@ -2847,6 +2930,14 @@ def get_page_ranges( self, offset: Optional[int] = None, length: Optional[int] = None, previous_snapshot_diff: Optional[Union[str, Dict[str, Any]]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]: """DEPRECATED: Returns the list of valid page ranges for a Page Blob or snapshot @@ -2918,7 +3009,15 @@ def get_page_ranges( offset=offset, length=length, previous_snapshot_diff=previous_snapshot_diff, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: if previous_snapshot_diff: ranges = self._client.page_blob.get_page_ranges_diff(**options) @@ -2930,11 +3029,18 @@ def get_page_ranges( @distributed_trace def list_page_ranges( - self, - *, + self, *, offset: Optional[int] = None, length: Optional[int] = None, previous_snapshot: Optional[Union[str, Dict[str, Any]]] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + results_per_page: Optional[int] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> ItemPaged[PageRange]: """Returns the list of valid page ranges for a Page Blob or snapshot @@ -2999,31 +3105,50 @@ def list_page_ranges( :returns: An iterable (auto-paging) of PageRange. :rtype: ~azure.core.paging.ItemPaged[~azure.storage.blob.PageRange] """ - results_per_page = kwargs.pop('results_per_page', None) options = _get_page_ranges_options( snapshot=self.snapshot, offset=offset, length=length, previous_snapshot_diff=previous_snapshot, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + results_per_page=results_per_page, + timeout=timeout, + **kwargs + ) if previous_snapshot: command = partial( self._client.page_blob.get_page_ranges_diff, - **options) + **options + ) else: command = partial( self._client.page_blob.get_page_ranges, - **options) + **options + ) return ItemPaged( - command, results_per_page=results_per_page, - page_iterator_class=PageRangePaged) + command, + results_per_page=results_per_page, + page_iterator_class=PageRangePaged + ) @distributed_trace def get_page_range_diff_for_managed_disk( self, previous_snapshot_url: str, offset: Optional[int] = None, - length:Optional[int] = None, + length: Optional[int] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]: """Returns the list of valid page ranges for a managed disk or snapshot. @@ -3089,7 +3214,14 @@ def get_page_range_diff_for_managed_disk( offset=offset, length=length, prev_snapshot_url=previous_snapshot_url, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + timeout=timeout, + **kwargs + ) try: ranges = self._client.page_blob.get_page_ranges_diff(**options) except HttpResponseError as error: @@ -3100,6 +3232,14 @@ def get_page_range_diff_for_managed_disk( def set_sequence_number( self, sequence_number_action: Union[str, "SequenceNumberAction"], sequence_number: Optional[str] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Sets the blob sequence number. @@ -3147,14 +3287,37 @@ def set_sequence_number( :returns: Blob-updated property dict (Etag and last modified). :rtype: Dict[str, Any] """ - options = _set_sequence_number_options(sequence_number_action, sequence_number=sequence_number, **kwargs) + options = _set_sequence_number_options( + sequence_number_action, + sequence_number=sequence_number, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.page_blob.update_sequence_number(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, datetime]]: + def resize_blob( + self, size: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime]]: """Resizes a page blob to the specified size. If the specified value is less than the current size of the blob, @@ -3205,7 +3368,18 @@ def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, datetime """ if kwargs.get('cpk') and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _resize_blob_options(size=size, **kwargs) + options = _resize_blob_options( + size=size, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + premium_page_blob_tier=premium_page_blob_tier, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.page_blob.resize(**options)) except HttpResponseError as error: @@ -3216,6 +3390,21 @@ def upload_page( self, page: bytes, offset: int, length: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + validate_content: Optional[bool] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + encoding: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """The Upload Pages operation writes a range of pages to a page blob. @@ -3301,13 +3490,28 @@ def upload_page( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_page_options( page=page, offset=offset, length=length, - **kwargs) + lease=lease, + validate_content=validate_content, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + encoding=encoding, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.page_blob.upload_pages(**options)) except HttpResponseError as error: @@ -3319,6 +3523,25 @@ def upload_pages_from_url( offset: int, length: int, source_offset: int, + *, + source_content_md5: Optional[bytes] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """ @@ -3409,27 +3632,45 @@ def upload_pages_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Response after uploading pages from specified URL. :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_pages_from_url_options( source_url=source_url, offset=offset, length=length, source_offset=source_offset, + source_content_md5=source_content_md5, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + lease=lease, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, **kwargs ) try: @@ -3438,7 +3679,23 @@ def upload_pages_from_url( process_storage_error(error) @distributed_trace - def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, Union[str, datetime]]: + def clear_page( + self, offset: int, + length: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime]]: """Clears a range of pages. :param int offset: @@ -3503,11 +3760,22 @@ def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, Union """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _clear_page_options( offset=offset, length=length, + lease=lease, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + timeout=timeout, **kwargs ) try: @@ -3519,6 +3787,20 @@ def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, Union def append_block( self, data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], length: Optional[int] = None, + *, + validate_content: Optional[bool] = None, + maxsize_condition: Optional[int] = None, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + encoding: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime, int]]: """Commits a new block of data to the end of the existing append blob. @@ -3600,11 +3882,24 @@ def append_block( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _append_block_options( data=data, length=length, + validate_content=validate_content, + maxsize_condition=maxsize_condition, + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + encoding=encoding, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, **kwargs ) try: @@ -3617,6 +3912,24 @@ def append_block_from_url( self, copy_source_url: str, source_offset: Optional[int] = None, source_length: Optional[int] = None, + *, + source_content_md5: Optional[bytearray] = None, + maxsize_condition: Optional[int] = None, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime, int]]: """ @@ -3701,26 +4014,43 @@ def append_block_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Result after appending a new block. :rtype: Dict[str, Union[str, datetime, int]] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _append_block_from_url_options( copy_source_url=copy_source_url, source_offset=source_offset, source_length=source_length, + source_content_md5=source_content_md5, + maxsize_condition=maxsize_condition, + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, **kwargs ) try: @@ -3730,7 +4060,17 @@ def append_block_from_url( process_storage_error(error) @distributed_trace - def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime, int]]: + def seal_append_blob( + self, *, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime, int]]: """The Seal operation seals the Append Blob to make it read-only. .. versionadded:: 12.4.0 @@ -3773,7 +4113,16 @@ def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime, int] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - options = _seal_append_blob_options(**kwargs) + options = _seal_append_blob_options( + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.append_blob.seal(**options)) except HttpResponseError as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index ab86e04fbd99..07ea70205dd9 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -1394,3 +1394,18 @@ def _set_standard_blob_tier_options( def _acquire_lease_options(**kwargs: Any): return {k: v for k, v in kwargs.items() if v is not None} + + +def _get_block_list_options(block_list_type: str, snapshot: Optional[str], **kwargs: Any) -> Dict[str, Any]: + access_conditions = get_access_conditions(kwargs.pop('lease', None)) + mod_conditions = get_modify_conditions(kwargs) + + options = { + 'list_type': block_list_type, + 'timeout': kwargs.pop('timeout', None), + 'lease_access_conditions': access_conditions, + 'modified_access_conditions': mod_conditions, + 'snapshot': snapshot, + } + options.update({k: v for k, v in kwargs.items() if v is not None}) + return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index ee55f6088448..e6cb8ea826a8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -46,6 +46,7 @@ _from_blob_url, _get_blob_properties_options, _get_blob_tags_options, + _get_block_list_options, _get_block_list_result, _get_page_ranges_options, _parse_url, @@ -2354,6 +2355,12 @@ async def stage_block_from_url( source_offset: Optional[int] = None, source_length: Optional[int] = None, source_content_md5: Optional[Union[bytes, bytearray]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Creates a new block to be committed as part of a blob where @@ -2387,15 +2394,15 @@ async def stage_block_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Blob property dict. :rtype: Dict[str, Any] """ @@ -2407,7 +2414,13 @@ async def stage_block_from_url( source_offset=source_offset, source_length=source_length, source_content_md5=source_content_md5, - **kwargs) + lease=lease, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.block_blob.stage_block_from_url(**options)) except HttpResponseError as error: @@ -2416,6 +2429,10 @@ async def stage_block_from_url( @distributed_trace_async async def get_block_list( self, block_list_type: str = "committed", + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[BlobBlock], List[BlobBlock]]: """The Get Block List operation retrieves the list of blocks that have @@ -2444,16 +2461,16 @@ async def get_block_list( :returns: A tuple of two lists - committed and uncommitted blocks :rtype: Tuple[List[BlobBlock], List[BlobBlock]] """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - mod_conditions = get_modify_conditions(kwargs) + options = _get_block_list_options( + block_list_type=block_list_type, + snapshot=self.snapshot, + lease=lease, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: - blocks = await self._client.block_blob.get_block_list( - list_type=block_list_type, - snapshot=self.snapshot, - timeout=kwargs.pop('timeout', None), - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, - **kwargs) + blocks = self._client.block_blob.get_block_list(**options) except HttpResponseError as error: process_storage_error(error) return _get_block_list_result(blocks) @@ -2463,6 +2480,21 @@ async def commit_block_list( self, block_list: List[BlobBlock], content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, + *, + tags: Optional[Dict[str, str]] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + immutability_policy: Optional["ImmutabilityPolicy"] = None, + legal_hold: Optional[bool] = None, + validate_content: Optional[bool] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + standard_blob_tier: Optional["StandardBlobTier"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """The Commit Block List operation writes a blob by specifying the list of @@ -2559,20 +2591,41 @@ async def commit_block_list( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _commit_block_list_options( block_list=block_list, content_settings=content_settings, metadata=metadata, - **kwargs) + tags=tags, + lease=lease, + immutability_policy=immutability_policy, + legal_hold=legal_hold, + validate_content=validate_content, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + standard_blob_tier=standard_blob_tier, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.block_blob.commit_block_list(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace_async - async def set_premium_page_blob_tier(self, premium_page_blob_tier: "PremiumPageBlobTier", **kwargs: Any) -> None: + async def set_premium_page_blob_tier( + self, premium_page_blob_tier: "PremiumPageBlobTier", + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> None: """Sets the page blob tiers on the blob. This API is only supported for page blobs on premium accounts. :param premium_page_blob_tier: @@ -2586,34 +2639,44 @@ async def set_premium_page_blob_tier(self, premium_page_blob_tier: "PremiumPageB .. versionadded:: 12.4.0 + :keyword lease: + Required if the blob has an active lease. Value can be a BlobLeaseClient object + or the lease ID as a string. + :paramtype lease: ~azure.storage.blob.aio.BlobLeaseClient or str :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword lease: - Required if the blob has an active lease. Value can be a BlobLeaseClient object - or the lease ID as a string. - :paramtype lease: ~azure.storage.blob.aio.BlobLeaseClient or str :rtype: None """ - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(lease) mod_conditions = get_modify_conditions(kwargs) if premium_page_blob_tier is None: raise ValueError("A PremiumPageBlobTiermust be specified") try: await self._client.blob.set_tier( tier=premium_page_blob_tier, - timeout=kwargs.pop('timeout', None), + timeout=timeout, lease_access_conditions=access_conditions, modified_access_conditions=mod_conditions, - **kwargs) + **kwargs + ) except HttpResponseError as error: process_storage_error(error) @distributed_trace_async - async def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]: + async def set_blob_tags( + self, tags: Optional[Dict[str, str]] = None, + *, + version_id: Optional[str] = None, + validate_content: Optional[bool] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Any]: """The Set Tags operation enables users to set tags on a blob or specific blob version, but not snapshot. Each call to this operation replaces all existing tags attached to the blob. To remove all tags from the blob, call this operation with no tags set. @@ -2654,15 +2717,29 @@ async def set_blob_tags(self, tags: Optional[Dict[str, str]] = None, **kwargs: A :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Any] """ - version_id = get_version_id(self.version_id, kwargs) - options = _set_blob_tags_options(version_id=version_id, tags=tags, **kwargs) + options = _set_blob_tags_options( + version_id=version_id or self.version_id, + tags=tags, + validate_content=validate_content, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.blob.set_tags(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace_async - async def get_blob_tags(self, **kwargs: Any) -> Dict[str, str]: + async def get_blob_tags( + self, *, + version_id: Optional[str] = None, + if_tags_match_condition: Optional[str] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, str]: """The Get Tags operation enables users to get tags on a blob or specific blob version, but not snapshot. .. versionadded:: 12.4.0 @@ -2687,8 +2764,14 @@ async def get_blob_tags(self, **kwargs: Any) -> Dict[str, str]: :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - version_id = get_version_id(self.version_id, kwargs) - options = _get_blob_tags_options(version_id=version_id, snapshot=self.snapshot, **kwargs) + options = _get_blob_tags_options( + version_id=version_id or self.version_id, + snapshot=self.snapshot, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: _, tags = await self._client.blob.get_tags(**options) return cast(Dict[str, str], parse_tags(tags)) @@ -2700,6 +2783,14 @@ async def get_page_ranges( self, offset: Optional[int] = None, length: Optional[int] = None, previous_snapshot_diff: Optional[Union[str, Dict[str, Any]]] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]: """DEPRECATED: Returns the list of valid page ranges for a Page Blob or snapshot @@ -2771,7 +2862,15 @@ async def get_page_ranges( offset=offset, length=length, previous_snapshot_diff=previous_snapshot_diff, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: if previous_snapshot_diff: ranges = await self._client.page_blob.get_page_ranges_diff(**options) @@ -2783,11 +2882,18 @@ async def get_page_ranges( @distributed_trace def list_page_ranges( - self, - *, + self, *, offset: Optional[int] = None, length: Optional[int] = None, previous_snapshot: Optional[Union[str, Dict[str, Any]]] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + results_per_page: Optional[int] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> AsyncItemPaged[PageRange]: """Returns the list of valid page ranges for a Page Blob or snapshot @@ -2852,31 +2958,50 @@ def list_page_ranges( :returns: An iterable (auto-paging) of PageRange. :rtype: ~azure.core.paging.ItemPaged[~azure.storage.blob.PageRange] """ - results_per_page = kwargs.pop('results_per_page', None) options = _get_page_ranges_options( snapshot=self.snapshot, offset=offset, length=length, previous_snapshot_diff=previous_snapshot, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + results_per_page=results_per_page, + timeout=timeout, + **kwargs + ) if previous_snapshot: command = partial( self._client.page_blob.get_page_ranges_diff, - **options) + **options + ) else: command = partial( self._client.page_blob.get_page_ranges, - **options) + **options + ) return AsyncItemPaged( - command, results_per_page=results_per_page, - page_iterator_class=PageRangePaged) + command, + results_per_page=results_per_page, + page_iterator_class=PageRangePaged + ) @distributed_trace_async async def get_page_range_diff_for_managed_disk( self, previous_snapshot_url: str, offset: Optional[int] = None, length: Optional[int] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Tuple[List[Dict[str, int]], List[Dict[str, int]]]: """Returns the list of valid page ranges for a managed disk or snapshot. @@ -2942,7 +3067,14 @@ async def get_page_range_diff_for_managed_disk( offset=offset, length=length, prev_snapshot_url=previous_snapshot_url, - **kwargs) + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + timeout=timeout, + **kwargs + ) try: ranges = await self._client.page_blob.get_page_ranges_diff(**options) except HttpResponseError as error: @@ -2953,6 +3085,14 @@ async def get_page_range_diff_for_managed_disk( async def set_sequence_number( self, sequence_number_action: Union[str, "SequenceNumberAction"], sequence_number: Optional[str] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """Sets the blob sequence number. @@ -3000,14 +3140,37 @@ async def set_sequence_number( :returns: Blob-updated property dict (Etag and last modified). :rtype: Dict[str, Any] """ - options = _set_sequence_number_options(sequence_number_action, sequence_number=sequence_number, **kwargs) + options = _set_sequence_number_options( + sequence_number_action, + sequence_number=sequence_number, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.page_blob.update_sequence_number(**options)) except HttpResponseError as error: process_storage_error(error) @distributed_trace_async - async def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, datetime]]: + async def resize_blob( + self, size: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime]]: """Resizes a page blob to the specified size. If the specified value is less than the current size of the blob, @@ -3058,7 +3221,18 @@ async def resize_blob(self, size: int, **kwargs: Any) -> Dict[str, Union[str, da """ if kwargs.get('cpk') and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") - options = _resize_blob_options(size=size, **kwargs) + options = _resize_blob_options( + size=size, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + premium_page_blob_tier=premium_page_blob_tier, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.page_blob.resize(**options)) except HttpResponseError as error: @@ -3069,6 +3243,21 @@ async def upload_page( self, page: bytes, offset: int, length: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + validate_content: Optional[bool] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + encoding: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime]]: """The Upload Pages operation writes a range of pages to a page blob. @@ -3154,13 +3343,28 @@ async def upload_page( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_page_options( page=page, offset=offset, length=length, - **kwargs) + lease=lease, + validate_content=validate_content, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + encoding=encoding, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.page_blob.upload_pages(**options)) except HttpResponseError as error: @@ -3172,6 +3376,25 @@ async def upload_pages_from_url( offset: int, length: int, source_offset: int, + *, + source_content_md5: Optional[bytes] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """ @@ -3262,28 +3485,46 @@ async def upload_pages_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Response after uploading pages from specified URL. :rtype: Dict[str, Any] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _upload_pages_from_url_options( source_url=source_url, offset=offset, length=length, source_offset=source_offset, + source_content_md5=source_content_md5, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + lease=lease, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, **kwargs ) try: @@ -3292,7 +3533,23 @@ async def upload_pages_from_url( process_storage_error(error) @distributed_trace_async - async def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, Union[str, datetime]]: + async def clear_page( + self, offset: int, + length: int, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_sequence_number_lte: Optional[int] = None, + if_sequence_number_lt: Optional[int] = None, + if_sequence_number_eq: Optional[int] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime]]: """Clears a range of pages. :param int offset: @@ -3357,11 +3614,22 @@ async def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _clear_page_options( offset=offset, length=length, + lease=lease, + if_sequence_number_lte=if_sequence_number_lte, + if_sequence_number_lt=if_sequence_number_lt, + if_sequence_number_eq=if_sequence_number_eq, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + cpk=cpk, + timeout=timeout, **kwargs ) try: @@ -3373,6 +3641,20 @@ async def clear_page(self, offset: int, length: int, **kwargs: Any) -> Dict[str, async def append_block( self, data: Union[bytes, str, Iterable[AnyStr], IO[AnyStr]], length: Optional[int] = None, + *, + validate_content: Optional[bool] = None, + maxsize_condition: Optional[int] = None, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + encoding: Optional[str] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime, int]]: """Commits a new block of data to the end of the existing append blob. @@ -3454,11 +3736,24 @@ async def append_block( """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _append_block_options( data=data, length=length, + validate_content=validate_content, + maxsize_condition=maxsize_condition, + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + encoding=encoding, + cpk=cpk, + encryption_scope=encryption_scope, + timeout=timeout, **kwargs ) try: @@ -3471,6 +3766,24 @@ async def append_block_from_url( self, copy_source_url: str, source_offset: Optional[int] = None, source_length: Optional[int] = None, + *, + source_content_md5: Optional[bytearray] = None, + maxsize_condition: Optional[int] = None, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + source_if_modified_since: Optional[datetime] = None, + source_if_unmodified_since: Optional[datetime] = None, + source_etag: Optional[str] = None, + source_match_condition: Optional["MatchConditions"] = None, + cpk: Optional["CustomerProvidedEncryptionKey"] = None, + encryption_scope: Optional[str] = None, + source_authorization: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Union[str, datetime, int]]: """ @@ -3555,26 +3868,43 @@ async def append_block_from_url( .. versionadded:: 12.2.0 + :keyword str source_authorization: + Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is + the prefix of the source_authorization string. :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. This value is not tracked or validated on the client. To configure client-side network timesouts see `here `__. - :keyword str source_authorization: - Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is - the prefix of the source_authorization string. :returns: Result after appending a new block. :rtype: Dict[str, Union[str, datetime, int]] """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - if kwargs.get('cpk') and self.scheme.lower() != 'https': + if cpk and self.scheme.lower() != 'https': raise ValueError("Customer provided encryption key must be used over HTTPS.") options = _append_block_from_url_options( copy_source_url=copy_source_url, source_offset=source_offset, source_length=source_length, + source_content_md5=source_content_md5, + maxsize_condition=maxsize_condition, + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_etag=source_etag, + source_match_condition=source_match_condition, + cpk=cpk, + encryption_scope=encryption_scope, + source_authorization=source_authorization, + timeout=timeout, **kwargs ) try: @@ -3584,7 +3914,17 @@ async def append_block_from_url( process_storage_error(error) @distributed_trace_async - async def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime, int]]: + async def seal_append_blob( + self, *, + appendpos_condition: Optional[int] = None, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Union[str, datetime, int]]: """The Seal operation seals the Append Blob to make it read-only. .. versionadded:: 12.4.0 @@ -3627,7 +3967,16 @@ async def seal_append_blob(self, **kwargs: Any) -> Dict[str, Union[str, datetime """ if self.require_encryption or (self.key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - options = _seal_append_blob_options(**kwargs) + options = _seal_append_blob_options( + appendpos_condition=appendpos_condition, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.append_blob.seal(**options)) except HttpResponseError as error: From f7ad368bbec1038fb1a13c5a7c58e2a936bc1fed Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 9 Apr 2025 15:55:08 -0400 Subject: [PATCH 38/43] Fixed pylint and mypy errors --- .../azure/storage/blob/_blob_client.py | 105 +++++++++++++++--- .../storage/blob/_blob_client_helpers.py | 14 +++ .../azure/storage/blob/_serialize.py | 5 +- .../storage/blob/aio/_blob_client_async.py | 98 +++++++++++++--- 4 files changed, 189 insertions(+), 33 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 1b96bce75f74..7cd82f0618e3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -45,6 +45,7 @@ _set_blob_metadata_options, _set_blob_tags_options, _set_http_headers_options, + _set_premium_page_blob_tier_options, _set_sequence_number_options, _set_standard_blob_tier_options, _stage_block_from_url_options, @@ -76,12 +77,7 @@ from ._quick_query_helper import BlobQueryReader from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper from ._shared.response_handlers import process_storage_error, return_response_headers -from ._serialize import ( - get_access_conditions, - get_api_version, - get_modify_conditions, - get_version_id -) +from ._serialize import get_api_version from ._upload_helpers import ( upload_append_blob, upload_block_blob, @@ -290,11 +286,33 @@ def from_blob_url( The optional blob snapshot on which to operate. This can be the snapshot ID string or the response returned from :func:`create_snapshot`. If specified, this will override the snapshot in the url. + :keyword str api_version: + The Storage API version to use for requests. Default value is the most recent service version that is + compatible with the current SDK. Setting to an older version may result in reduced feature compatibility. + + .. versionadded:: 12.2.0 + + :keyword str secondary_hostname: + The hostname of the secondary endpoint. :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. :keyword str audience: The audience to use when requesting tokens for Azure Active Directory authentication. Only has an effect when credential is of type TokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. + :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks. + Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_put_size: + If the blob size is less than or equal max_single_put_size, then the blob will be + uploaded with only one http PUT request. If the blob size is larger than max_single_put_size, + the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB. + :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient + algorithm when uploading a block blob. Defaults to 4*1024*1024+1. + :keyword bool use_byte_buffer: Use a byte buffer for block blob uploads. Defaults to False. + :keyword int max_page_size: The maximum chunk size for uploading a page blob. Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_get_size: The maximum size for a blob to be downloaded in a single call, + the exceeded part will be downloaded in chunks (could be parallel). Defaults to 32*1024*1024, or 32MB. + :keyword int max_chunk_get_size: The maximum chunk size used for downloading a blob. + Defaults to 4*1024*1024, or 4MB. :returns: A Blob client. :rtype: ~azure.storage.blob.BlobClient """ @@ -365,11 +383,33 @@ def from_connection_string( ~azure.core.credentials.AzureSasCredential or ~azure.core.credentials.TokenCredential or str or Dict[str, str] or None + :keyword str api_version: + The Storage API version to use for requests. Default value is the most recent service version that is + compatible with the current SDK. Setting to an older version may result in reduced feature compatibility. + + .. versionadded:: 12.2.0 + + :keyword str secondary_hostname: + The hostname of the secondary endpoint. :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. :keyword str audience: The audience to use when requesting tokens for Azure Active Directory authentication. Only has an effect when credential is of type TokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. + :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks. + Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_put_size: + If the blob size is less than or equal max_single_put_size, then the blob will be + uploaded with only one http PUT request. If the blob size is larger than max_single_put_size, + the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB. + :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient + algorithm when uploading a block blob. Defaults to 4*1024*1024+1. + :keyword bool use_byte_buffer: Use a byte buffer for block blob uploads. Defaults to False. + :keyword int max_page_size: The maximum chunk size for uploading a page blob. Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_get_size: The maximum size for a blob to be downloaded in a single call, + the exceeded part will be downloaded in chunks (could be parallel). Defaults to 32*1024*1024, or 32MB. + :keyword int max_chunk_get_size: The maximum chunk size used for downloading a blob. + Defaults to 4*1024*1024, or 4MB. :returns: A Blob client. :rtype: ~azure.storage.blob.BlobClient @@ -1396,7 +1436,18 @@ def get_blob_properties( return blob_props @distributed_trace - def set_http_headers(self, content_settings: Optional["ContentSettings"] = None, **kwargs: Any) -> Dict[str, Any]: + def set_http_headers( + self, content_settings: Optional["ContentSettings"] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any + ) -> Dict[str, Any]: """Sets system properties on the blob. If one property is set for the content_settings, all properties will be overridden. @@ -1440,7 +1491,17 @@ def set_http_headers(self, content_settings: Optional["ContentSettings"] = None, :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Any] """ - options = _set_http_headers_options(content_settings=content_settings, **kwargs) + options = _set_http_headers_options( + content_settings=content_settings, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], self._client.blob.set_http_headers(**options)) except HttpResponseError as error: @@ -2015,6 +2076,7 @@ def start_copy_from_url( if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, destination_lease: Optional[Union[BlobLeaseClient, str]] = None, source_lease: Optional[Union[BlobLeaseClient, str]] = None, premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, @@ -2125,6 +2187,12 @@ def start_copy_from_url( and act according to the condition specified by the `match_condition` parameter. :keyword ~azure.core.MatchConditions source_match_condition: The source match condition to use upon the etag. + :keyword str if_tags_match_condition: + Specify a SQL where clause on blob tags to operate only on blob with a matching value. + eg. ``\"\\\"tagname\\\"='my tag'\"`` + + .. versionadded:: 12.4.0 + :keyword ~datetime.datetime if_modified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. @@ -2219,6 +2287,7 @@ def start_copy_from_url( if_unmodified_since=if_unmodified_since, etag=etag, match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, destination_lease=destination_lease, source_lease=source_lease, premium_page_blob_tier=premium_page_blob_tier, @@ -2403,7 +2472,7 @@ def set_standard_blob_tier( :rtype: None """ options = _set_standard_blob_tier_options( - version_id, + version_id or self.version_id, self.snapshot, standard_blob_tier=standard_blob_tier, timeout=timeout, @@ -2769,6 +2838,7 @@ def commit_block_list( def set_premium_page_blob_tier( self, premium_page_blob_tier: "PremiumPageBlobTier", *, + if_tags_match_condition: Optional[str] = None, lease: Optional[Union[BlobLeaseClient, str]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -2798,18 +2868,17 @@ def set_premium_page_blob_tier( #other-client--per-operation-configuration>`__. :rtype: None """ - access_conditions = get_access_conditions(lease) - mod_conditions = get_modify_conditions(kwargs) if premium_page_blob_tier is None: raise ValueError("A PremiumPageBlobTier must be specified") + options = _set_premium_page_blob_tier_options( + premium_page_blob_tier=premium_page_blob_tier, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: - self._client.blob.set_tier( - tier=premium_page_blob_tier, - timeout=timeout, - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, - **kwargs - ) + self._client.blob.set_tier(**options) except HttpResponseError as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 07ea70205dd9..0f90bf24026a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -1409,3 +1409,17 @@ def _get_block_list_options(block_list_type: str, snapshot: Optional[str], **kwa } options.update({k: v for k, v in kwargs.items() if v is not None}) return options + + +def _set_premium_page_blob_tier_options(**kwargs: Any) -> Dict[str, Any]: + access_conditions = get_access_conditions(kwargs.pop('lease', None)) + mod_conditions = get_modify_conditions(kwargs) + + options = { + 'tier': kwargs.pop('premium_page_blob_tier'), + 'timeout': kwargs.pop('timeout', None), + 'lease_access_conditions': access_conditions, + 'modified_access_conditions': mod_conditions, + } + options.update({k: v for k, v in kwargs.items() if v is not None}) + return options diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py index 56009ed68141..f9e6c3e14db6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: from ._lease import BlobLeaseClient + from aio._lease_async import BlobLeaseClient as AsyncBlobLeaseClient _SUPPORTED_API_VERSIONS = [ @@ -90,7 +91,9 @@ def _get_match_headers( return if_match, if_none_match -def get_access_conditions(lease: Optional[Union["BlobLeaseClient", str]]) -> Optional[LeaseAccessConditions]: +def get_access_conditions( + lease: Optional[Union["BlobLeaseClient", "AsyncBlobLeaseClient", str]] +) -> Optional[LeaseAccessConditions]: if lease is None: return None if hasattr(lease, "id"): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index e6cb8ea826a8..d837d04e00f7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -55,6 +55,7 @@ _set_blob_metadata_options, _set_blob_tags_options, _set_http_headers_options, + _set_premium_page_blob_tier_options, _set_sequence_number_options, _set_standard_blob_tier_options, _stage_block_from_url_options, @@ -74,7 +75,7 @@ from .._encryption import StorageEncryptionMixin, _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from .._generated.aio import AzureBlobStorage from .._models import BlobType, BlobBlock, BlobProperties, PageRange -from .._serialize import get_access_conditions, get_api_version, get_modify_conditions, get_version_id +from .._serialize import get_api_version from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper, parse_connection_str from .._shared.policies_async import ExponentialRetry from .._shared.response_handlers import process_storage_error, return_response_headers @@ -270,11 +271,33 @@ def from_blob_url( The optional blob snapshot on which to operate. This can be the snapshot ID string or the response returned from :func:`create_snapshot`. If specified, this will override the snapshot in the url. + :keyword str api_version: + The Storage API version to use for requests. Default value is the most recent service version that is + compatible with the current SDK. Setting to an older version may result in reduced feature compatibility. + + .. versionadded:: 12.2.0 + + :keyword str secondary_hostname: + The hostname of the secondary endpoint. :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. :keyword str audience: The audience to use when requesting tokens for Azure Active Directory authentication. Only has an effect when credential is of type TokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. + :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks. + Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_put_size: + If the blob size is less than or equal max_single_put_size, then the blob will be + uploaded with only one http PUT request. If the blob size is larger than max_single_put_size, + the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB. + :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient + algorithm when uploading a block blob. Defaults to 4*1024*1024+1. + :keyword bool use_byte_buffer: Use a byte buffer for block blob uploads. Defaults to False. + :keyword int max_page_size: The maximum chunk size for uploading a page blob. Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_get_size: The maximum size for a blob to be downloaded in a single call, + the exceeded part will be downloaded in chunks (could be parallel). Defaults to 32*1024*1024, or 32MB. + :keyword int max_chunk_get_size: The maximum chunk size used for downloading a blob. + Defaults to 4*1024*1024, or 4MB. :returns: A Blob client. :rtype: ~azure.storage.blob.BlobClient """ @@ -341,11 +364,33 @@ def from_connection_string( If using an instance of AzureNamedKeyCredential, "name" should be the storage account name, and "key" should be the storage account key. :type credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] # pylint: disable=line-too-long + :keyword str api_version: + The Storage API version to use for requests. Default value is the most recent service version that is + compatible with the current SDK. Setting to an older version may result in reduced feature compatibility. + + .. versionadded:: 12.2.0 + + :keyword str secondary_hostname: + The hostname of the secondary endpoint. :keyword str version_id: The version id parameter is an opaque DateTime value that, when present, specifies the version of the blob to operate on. :keyword str audience: The audience to use when requesting tokens for Azure Active Directory authentication. Only has an effect when credential is of type TokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. + :keyword int max_block_size: The maximum chunk size for uploading a block blob in chunks. + Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_put_size: + If the blob size is less than or equal max_single_put_size, then the blob will be + uploaded with only one http PUT request. If the blob size is larger than max_single_put_size, + the blob will be uploaded in chunks. Defaults to 64*1024*1024, or 64MB. + :keyword int min_large_block_upload_threshold: The minimum chunk size required to use the memory efficient + algorithm when uploading a block blob. Defaults to 4*1024*1024+1. + :keyword bool use_byte_buffer: Use a byte buffer for block blob uploads. Defaults to False. + :keyword int max_page_size: The maximum chunk size for uploading a page blob. Defaults to 4*1024*1024, or 4MB. + :keyword int max_single_get_size: The maximum size for a blob to be downloaded in a single call, + the exceeded part will be downloaded in chunks (could be parallel). Defaults to 32*1024*1024, or 32MB. + :keyword int max_chunk_get_size: The maximum chunk size used for downloading a blob. + Defaults to 4*1024*1024, or 4MB. :returns: A Blob client. :rtype: ~azure.storage.blob.BlobClient @@ -1245,6 +1290,14 @@ async def get_blob_properties( @distributed_trace_async async def set_http_headers( self, content_settings: Optional["ContentSettings"] = None, + *, + lease: Optional[Union[BlobLeaseClient, str]] = None, + if_modified_since: Optional[datetime] = None, + if_unmodified_since: Optional[datetime] = None, + etag: Optional[str] = None, + match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, + timeout: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Sets system properties on the blob. @@ -1290,7 +1343,17 @@ async def set_http_headers( :returns: Blob-updated property dict (Etag and last modified) :rtype: Dict[str, Any] """ - options = _set_http_headers_options(content_settings=content_settings, **kwargs) + options = _set_http_headers_options( + content_settings=content_settings, + lease=lease, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since, + etag=etag, + match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, + timeout=timeout, + **kwargs + ) try: return cast(Dict[str, Any], await self._client.blob.set_http_headers(**options)) except HttpResponseError as error: @@ -1418,8 +1481,6 @@ async def set_immutability_policy( :returns: Key value pairs of blob tags. :rtype: Dict[str, str] """ - - version_id = get_version_id(self.version_id, kwargs) kwargs['immutability_policy_expiry'] = immutability_policy.expiry_time kwargs['immutability_policy_mode'] = immutability_policy.policy_mode return cast(Dict[str, str], await self._client.blob.set_immutability_policy( @@ -1868,6 +1929,7 @@ async def start_copy_from_url( if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, match_condition: Optional["MatchConditions"] = None, + if_tags_match_condition: Optional[str] = None, destination_lease: Optional[Union[BlobLeaseClient, str]] = None, source_lease: Optional[Union[BlobLeaseClient, str]] = None, premium_page_blob_tier: Optional["PremiumPageBlobTier"] = None, @@ -2077,6 +2139,7 @@ async def start_copy_from_url( if_unmodified_since=if_unmodified_since, etag=etag, match_condition=match_condition, + if_tags_match_condition=if_tags_match_condition, destination_lease=destination_lease, source_lease=source_lease, premium_page_blob_tier=premium_page_blob_tier, @@ -2235,6 +2298,13 @@ async def set_standard_blob_tier( :type standard_blob_tier: str or ~azure.storage.blob.StandardBlobTier :keyword ~azure.storage.blob.RehydratePriority rehydrate_priority: Indicates the priority with which to rehydrate an archived blob + :keyword str version_id: + The version id parameter is an opaque DateTime + value that, when present, specifies the version of the blob to download. + + .. versionadded:: 12.4.0 + + This keyword argument was introduced in API version '2019-12-12'. :keyword str if_tags_match_condition: Specify a SQL where clause on blob tags to operate only on blob with a matching value. eg. ``\"\\\"tagname\\\"='my tag'\"`` @@ -2254,7 +2324,7 @@ async def set_standard_blob_tier( :rtype: None """ options = _set_standard_blob_tier_options( - version_id, + version_id or self.version_id, self.snapshot, standard_blob_tier=standard_blob_tier, timeout=timeout, @@ -2622,6 +2692,7 @@ async def commit_block_list( async def set_premium_page_blob_tier( self, premium_page_blob_tier: "PremiumPageBlobTier", *, + if_tags_match_condition: Optional[str] = None, lease: Optional[Union[BlobLeaseClient, str]] = None, timeout: Optional[int] = None, **kwargs: Any @@ -2651,18 +2722,17 @@ async def set_premium_page_blob_tier( #other-client--per-operation-configuration>`__. :rtype: None """ - access_conditions = get_access_conditions(lease) - mod_conditions = get_modify_conditions(kwargs) if premium_page_blob_tier is None: raise ValueError("A PremiumPageBlobTiermust be specified") + options = _set_premium_page_blob_tier_options( + premium_page_blob_tier=premium_page_blob_tier, + if_tags_match_condition=if_tags_match_condition, + lease=lease, + timeout=timeout, + **kwargs + ) try: - await self._client.blob.set_tier( - tier=premium_page_blob_tier, - timeout=timeout, - lease_access_conditions=access_conditions, - modified_access_conditions=mod_conditions, - **kwargs - ) + await self._client.blob.set_tier(**options) except HttpResponseError as error: process_storage_error(error) From 8de799aa998f0338bd5de58dc76a1b954a7f9cba Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 9 Apr 2025 16:02:02 -0400 Subject: [PATCH 39/43] Fixed missing await --- .../azure/storage/blob/aio/_blob_client_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index d837d04e00f7..7ea481b803a6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -2540,7 +2540,7 @@ async def get_block_list( **kwargs ) try: - blocks = self._client.block_blob.get_block_list(**options) + blocks = await self._client.block_blob.get_block_list(**options) except HttpResponseError as error: process_storage_error(error) return _get_block_list_result(blocks) From d8aebd04f19582de5f0ae6427946e419a01428f9 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 9 Apr 2025 16:48:26 -0400 Subject: [PATCH 40/43] Fixed test failures --- .../azure-storage-blob/azure/storage/blob/_blob_client.py | 5 +---- .../azure/storage/blob/_blob_client_helpers.py | 2 +- .../azure/storage/blob/aio/_blob_client_async.py | 5 +---- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 7cd82f0618e3..6b4f1d3b9203 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -423,8 +423,6 @@ def from_connection_string( :caption: Creating the BlobClient from a connection string. """ account_url, secondary, credential = parse_connection_str(conn_str, credential, 'blob') - if 'secondary_hostname' not in kwargs: - kwargs['secondary_hostname'] = secondary return cls( account_url, container_name=container_name, @@ -434,7 +432,7 @@ def from_connection_string( version_id=version_id, audience=audience, api_version=api_version, - secondary_hostname=secondary_hostname, + secondary_hostname=secondary or secondary_hostname, max_block_size=max_block_size, max_page_size=max_page_size, max_chunk_get_size=max_chunk_get_size, @@ -3185,7 +3183,6 @@ def list_page_ranges( etag=etag, match_condition=match_condition, if_tags_match_condition=if_tags_match_condition, - results_per_page=results_per_page, timeout=timeout, **kwargs ) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 0f90bf24026a..f00ae4089921 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -226,7 +226,7 @@ def _upload_blob_from_url_options(source_url: str, **kwargs: Any) -> Dict[str, A options = { 'copy_source_authorization': source_authorization, 'content_length': 0, - 'copy_source_blob_properties': kwargs.pop('include_source_blob_properties') or True, + 'copy_source_blob_properties': kwargs.pop('include_source_blob_properties'), 'source_content_md5': kwargs.pop('source_content_md5', None), 'copy_source': source_url, 'modified_access_conditions': get_modify_conditions(kwargs), diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 7ea481b803a6..21f34fb47119 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -404,8 +404,6 @@ def from_connection_string( :caption: Creating the BlobClient from a connection string. """ account_url, secondary, credential = parse_connection_str(conn_str, credential, 'blob') - if 'secondary_hostname' not in kwargs: - kwargs['secondary_hostname'] = secondary return cls( account_url, container_name=container_name, @@ -415,7 +413,7 @@ def from_connection_string( version_id=version_id, audience=audience, api_version=api_version, - secondary_hostname=secondary_hostname, + secondary_hostname=secondary or secondary_hostname, max_block_size=max_block_size, max_page_size=max_page_size, max_chunk_get_size=max_chunk_get_size, @@ -3039,7 +3037,6 @@ def list_page_ranges( etag=etag, match_condition=match_condition, if_tags_match_condition=if_tags_match_condition, - results_per_page=results_per_page, timeout=timeout, **kwargs ) From b6bf76814b207b47b227a5f61a89ad202759750c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 17 Apr 2025 12:57:58 -0400 Subject: [PATCH 41/43] PR feedback --- .../azure/storage/blob/_blob_client.py | 6 +++--- .../azure/storage/blob/_blob_client_helpers.py | 12 ++++++------ .../azure/storage/blob/aio/_blob_client_async.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 6b4f1d3b9203..5983c99eaa76 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -432,7 +432,7 @@ def from_connection_string( version_id=version_id, audience=audience, api_version=api_version, - secondary_hostname=secondary or secondary_hostname, + secondary_hostname=secondary_hostname or secondary, max_block_size=max_block_size, max_page_size=max_page_size, max_chunk_get_size=max_chunk_get_size, @@ -3064,7 +3064,7 @@ def get_page_ranges( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) + :rtype: Tuple[List[Dict[str, int]], List[Dict[str, int]]] """ warnings.warn( "get_page_ranges is deprecated, use list_page_ranges instead", @@ -3273,7 +3273,7 @@ def get_page_range_diff_for_managed_disk( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) + :rtype: Tuple[List[Dict[str, int]], List[Dict[str, int]]] """ options = _get_page_ranges_options( snapshot=self.snapshot, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index f00ae4089921..5d3c612407e0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -522,9 +522,9 @@ def _create_page_blob_options( tier = None if premium_page_blob_tier: try: - tier = premium_page_blob_tier.value # type: ignore + tier = premium_page_blob_tier.value # type: ignore [assignment] except AttributeError: - tier = premium_page_blob_tier # type: ignore + tier = premium_page_blob_tier # type: ignore [assignment] blob_tags_string = serialize_blob_tags_header(kwargs.pop('tags', None)) @@ -734,7 +734,7 @@ def _stage_block_options( ) -> Dict[str, Any]: block_id = encode_base64(str(block_id)) if isinstance(data, str): - data = data.encode(kwargs.pop('encoding') or 'UTF-8') # type: ignore + data = data.encode(kwargs.pop('encoding') or 'UTF-8') access_conditions = get_access_conditions(kwargs.pop('lease', None)) if length is None: length = get_length(data) @@ -955,10 +955,10 @@ def _get_page_ranges_options( } if previous_snapshot_diff: try: - options['prevsnapshot'] = previous_snapshot_diff.snapshot # type: ignore + options['prevsnapshot'] = previous_snapshot_diff.snapshot # type: ignore [assignment] except AttributeError: try: - options['prevsnapshot'] = previous_snapshot_diff['snapshot'] # type: ignore + options['prevsnapshot'] = previous_snapshot_diff['snapshot'] # type: ignore [assignment] except TypeError: options['prevsnapshot'] = previous_snapshot_diff options.update({k: v for k, v in kwargs.items() if v is not None}) @@ -1025,7 +1025,7 @@ def _upload_page_options( if length is None or length % 512 != 0: raise ValueError("length must be an integer that aligns with 512 page size") end_range = offset + length - 1 # Reformat to an inclusive range index - content_range = f'bytes={offset}-{end_range}' # type: ignore + content_range = f'bytes={offset}-{end_range}' access_conditions = get_access_conditions(kwargs.pop('lease', None)) seq_conditions = SequenceNumberAccessConditions( if_sequence_number_less_than_or_equal_to=kwargs.pop('if_sequence_number_lte', None), diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 21f34fb47119..1b44355c2207 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -413,7 +413,7 @@ def from_connection_string( version_id=version_id, audience=audience, api_version=api_version, - secondary_hostname=secondary or secondary_hostname, + secondary_hostname=secondary_hostname or secondary, max_block_size=max_block_size, max_page_size=max_page_size, max_chunk_get_size=max_chunk_get_size, @@ -2918,7 +2918,7 @@ async def get_page_ranges( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) + :rtype: Tuple[List[Dict[str, int]], List[Dict[str, int]]] """ warnings.warn( "get_page_ranges is deprecated, use list_page_ranges instead", @@ -3127,7 +3127,7 @@ async def get_page_range_diff_for_managed_disk( :returns: A tuple of two lists of page ranges as dictionaries with 'start' and 'end' keys. The first element are filled page ranges, the 2nd element is cleared page ranges. - :rtype: tuple(list(Dict[str, str], list(Dict[str, str]) + :rtype: Tuple[List[Dict[str, int]], List[Dict[str, int]]] """ options = _get_page_ranges_options( snapshot=self.snapshot, From 043c4ec2b2b837e31fdd769e7d330cc82913bb27 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 17 Apr 2025 14:10:38 -0400 Subject: [PATCH 42/43] Reverted couple of redundant changes --- .../azure/storage/blob/_blob_client_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index 5d3c612407e0..b7e40b826d58 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -135,10 +135,10 @@ def _upload_blob_options( # pylint:disable=too-many-statements raise TypeError(f"Unsupported data type: {type(data)}") validate_content = kwargs.pop('validate_content') or False - content_settings = kwargs.pop('content_settings') or None + content_settings = kwargs.pop('content_settings', None) overwrite = kwargs.pop('overwrite') or False max_concurrency = kwargs.pop('max_concurrency') or 1 - cpk = kwargs.pop('cpk') or None + cpk = kwargs.pop('cpk', None) cpk_info = None if cpk: cpk_info = CpkInfo( From ba17d5c6e98cc08484f76dae388cde5888ebec85 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 17 Apr 2025 14:45:44 -0400 Subject: [PATCH 43/43] specific type ignore --- .../azure/storage/blob/_blob_client_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py index b7e40b826d58..9c69a78fa44b 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py @@ -522,9 +522,9 @@ def _create_page_blob_options( tier = None if premium_page_blob_tier: try: - tier = premium_page_blob_tier.value # type: ignore [assignment] + tier = premium_page_blob_tier.value # type: ignore [attr-defined] except AttributeError: - tier = premium_page_blob_tier # type: ignore [assignment] + tier = premium_page_blob_tier # type: ignore [attr-defined] blob_tags_string = serialize_blob_tags_header(kwargs.pop('tags', None)) @@ -955,10 +955,10 @@ def _get_page_ranges_options( } if previous_snapshot_diff: try: - options['prevsnapshot'] = previous_snapshot_diff.snapshot # type: ignore [assignment] + options['prevsnapshot'] = previous_snapshot_diff.snapshot # type: ignore [union-attr] except AttributeError: try: - options['prevsnapshot'] = previous_snapshot_diff['snapshot'] # type: ignore [assignment] + options['prevsnapshot'] = previous_snapshot_diff['snapshot'] # type: ignore [union-attr, index] except TypeError: options['prevsnapshot'] = previous_snapshot_diff options.update({k: v for k, v in kwargs.items() if v is not None})