project: MultiCluster Object Storage (#416)

* Add warnings to deprecated OBJ API usage (#410)

* add deprecated

* add dependency

* fix deprecated

* Update OBJ Group and Objects for MultiCluster Object Storage API Changes (#426)

---------

Co-authored-by: Ye Chen <[email protected]>

Author: zliang-akamai

linode_api4/groups/object_storage.py

@@ -1,6 +1,10 @@
+import re
+import warnings
 from typing import List, Optional, Union
 from urllib import parse
 
+from deprecated import deprecated
+
 from linode_api4.errors import UnexpectedResponseError
 from linode_api4.groups import Group
 from linode_api4.objects import (
@@ -9,6 +13,7 @@
     ObjectStorageACL,
     ObjectStorageBucket,
     ObjectStorageCluster,
+    ObjectStorageKeyPermission,
     ObjectStorageKeys,
 )
 from linode_api4.util import drop_null_keys
@@ -20,8 +25,14 @@ class ObjectStorageGroup(Group):
     available clusters, buckets, and managing keys and TLS/SSL certs, etc.
     """
 
+    @deprecated(
+        reason="deprecated to use regions list API for listing available OJB clusters"
+    )
     def clusters(self, *filters):
         """
+        This endpoint will be deprecated to use the regions list API to list available OBJ clusters,
+        and a new access key API will directly expose the S3 endpoint hostname.
+
         Returns a list of available Object Storage Clusters.  You may filter
         this query to return only Clusters that are available in a specific region::
 
@@ -58,6 +69,7 @@ def keys_create(
         self,
         label: str,
         bucket_access: Optional[Union[dict, List[dict]]] = None,
+        regions: Optional[List[str]] = None,
     ):
         """
         Creates a new Object Storage keypair that may be used to interact directly
@@ -97,14 +109,16 @@ def keys_create(
 
         :param label: The label for this keypair, for identification only.
         :type label: str
-        :param bucket_access: One or a list of dicts with keys "cluster,"
-                              "permissions", and "bucket_name".  If given, the
-                              resulting Object Storage keys will only have the
-                              requested level of access to the requested buckets,
-                              if they exist and are owned by you.  See the provided
-                              :any:`bucket_access` function for a convenient way
-                              to create these dicts.
-        :type bucket_access: dict or list of dict
+        :param bucket_access: One or a list of dicts with keys "cluster," "region",
+                              "permissions", and "bucket_name". "cluster" key is
+                              deprecated because multiple cluster can be placed
+                              in the same region. Please consider switching to
+                              regions. If given, the resulting Object Storage keys
+                              will only have the requested level of access to the
+                              requested buckets, if they exist and are owned by
+                              you.  See the provided :any:`bucket_access` function
+                              for a convenient way to create these dicts.
+        :type bucket_access: Optional[Union[dict, List[dict]]]
 
         :returns: The new keypair, with the secret key populated.
         :rtype: ObjectStorageKeys
@@ -115,22 +129,35 @@ def keys_create(
             if not isinstance(bucket_access, list):
                 bucket_access = [bucket_access]
 
-            ba = [
-                {
-                    "permissions": c.get("permissions"),
-                    "bucket_name": c.get("bucket_name"),
-                    "cluster": (
-                        c.id
-                        if "cluster" in c
-                        and issubclass(type(c["cluster"]), Base)
-                        else c.get("cluster")
-                    ),
+            ba = []
+            for access_rule in bucket_access:
+                access_rule_json = {
+                    "permissions": access_rule.get("permissions"),
+                    "bucket_name": access_rule.get("bucket_name"),
                 }
-                for c in bucket_access
-            ]
+
+                if "region" in access_rule:
+                    access_rule_json["region"] = access_rule.get("region")
+                elif "cluster" in access_rule:
+                    warnings.warn(
+                        "'cluster' is a deprecated attribute, "
+                        "please consider using 'region' instead.",
+                        DeprecationWarning,
+                    )
+                    access_rule_json["cluster"] = (
+                        access_rule.id
+                        if "cluster" in access_rule
+                        and issubclass(type(access_rule["cluster"]), Base)
+                        else access_rule.get("cluster")
+                    )
+
+                ba.append(access_rule_json)
 
             params["bucket_access"] = ba
 
+        if regions is not None:
+            params["regions"] = regions
+
         result = self.client.post("/object-storage/keys", data=params)
 
         if not "id" in result:
@@ -142,9 +169,74 @@ def keys_create(
         ret = ObjectStorageKeys(self.client, result["id"], result)
         return ret
 
-    def bucket_access(self, cluster, bucket_name, permissions):
-        return ObjectStorageBucket.access(
-            self, cluster, bucket_name, permissions
+    @classmethod
+    def bucket_access(
+        cls,
+        cluster_or_region: str,
+        bucket_name: str,
+        permissions: Union[str, ObjectStorageKeyPermission],
+    ):
+        """
+        Returns a dict formatted to be included in the `bucket_access` argument
+        of :any:`keys_create`.  See the docs for that method for an example of
+        usage.
+
+        :param cluster_or_region: The region or Object Storage cluster to grant access in.
+        :type cluster_or_region: str
+        :param bucket_name: The name of the bucket to grant access to.
+        :type bucket_name: str
+        :param permissions: The permissions to grant.  Should be one of "read_only"
+                            or "read_write".
+        :type permissions: Union[str, ObjectStorageKeyPermission]
+        :param use_region: Whether to use region mode.
+        :type use_region: bool
+
+        :returns: A dict formatted correctly for specifying bucket access for
+                  new keys.
+        :rtype: dict
+        """
+
+        result = {
+            "bucket_name": bucket_name,
+            "permissions": permissions,
+        }
+
+        if cls.is_cluster(cluster_or_region):
+            warnings.warn(
+                "Cluster ID for Object Storage APIs has been deprecated. "
+                "Please consider switch to a region ID (e.g., from `us-mia-1` to `us-mia`)",
+                DeprecationWarning,
+            )
+            result["cluster"] = cluster_or_region
+        else:
+            result["region"] = cluster_or_region
+
+        return result
+
+    def buckets_in_region(self, region: str, *filters):
+        """
+        Returns a list of Buckets in the region belonging to this Account.
+
+        This endpoint is available for convenience.
+        It is recommended that instead you use the more fully-featured S3 API directly.
+
+        API Documentation: https://www.linode.com/docs/api/object-storage/#object-storage-buckets-in-cluster-list
+
+        :param filters: Any number of filters to apply to this query.
+                        See :doc:`Filtering Collections</linode_api4/objects/filtering>`
+                        for more details on filtering.
+
+        :param region: The ID of an object storage region (e.g. `us-mia-1`).
+        :type region: str
+
+        :returns: A list of Object Storage Buckets that in the requested cluster.
+        :rtype: PaginatedList of ObjectStorageBucket
+        """
+
+        return self.client._get_and_filter(
+            ObjectStorageBucket,
+            *filters,
+            endpoint=f"/object-storage/buckets/{region}",
         )
 
     def cancel(self):
@@ -197,10 +289,14 @@ def buckets(self, *filters):
         """
         return self.client._get_and_filter(ObjectStorageBucket, *filters)
 
+    @staticmethod
+    def is_cluster(cluster_or_region: str):
+        return bool(re.match(r"^[a-z]{2}-[a-z]+-[0-9]+$", cluster_or_region))
+
     def bucket_create(
         self,
-        cluster,
-        label,
+        cluster_or_region: Union[str, ObjectStorageCluster],
+        label: str,
         acl: ObjectStorageACL = ObjectStorageACL.PRIVATE,
         cors_enabled=False,
     ):
@@ -240,17 +336,30 @@ def bucket_create(
         :returns: A Object Storage Buckets that created by user.
         :rtype: ObjectStorageBucket
         """
-        cluster_id = (
-            cluster.id if isinstance(cluster, ObjectStorageCluster) else cluster
+        cluster_or_region_id = (
+            cluster_or_region.id
+            if isinstance(cluster_or_region, ObjectStorageCluster)
+            else cluster_or_region
         )
 
         params = {
-            "cluster": cluster_id,
             "label": label,
             "acl": acl,
             "cors_enabled": cors_enabled,
         }
 
+        if self.is_cluster(cluster_or_region_id):
+            warnings.warn(
+                "The cluster parameter has been deprecated for creating a object "
+                "storage bucket. Please consider switching to a region value. For "
+                "example, a cluster value of `us-mia-1` can be translated to a "
+                "region value of `us-mia`.",
+                DeprecationWarning,
+            )
+            params["cluster"] = cluster_or_region_id
+        else:
+            params["region"] = cluster_or_region_id
+
         result = self.client.post("/object-storage/buckets", data=params)
 
         if not "label" in result or not "cluster" in result:
@@ -263,21 +372,21 @@ def bucket_create(
             self.client, result["label"], result["cluster"], result
         )
 
-    def object_acl_config(self, cluster_id, bucket, name=None):
+    def object_acl_config(self, cluster_or_region_id: str, bucket, name=None):
         return ObjectStorageBucket(
-            self.client, bucket, cluster_id
+            self.client, bucket, cluster_or_region_id
         ).object_acl_config(name)
 
     def object_acl_config_update(
-        self, cluster_id, bucket, acl: ObjectStorageACL, name
+        self, cluster_or_region_id, bucket, acl: ObjectStorageACL, name
     ):
         return ObjectStorageBucket(
-            self.client, bucket, cluster_id
+            self.client, bucket, cluster_or_region_id
         ).object_acl_config_update(acl, name)
 
     def object_url_create(
         self,
-        cluster_id,
+        cluster_or_region_id,
         bucket,
         method,
         name,
@@ -294,8 +403,8 @@ def object_url_create(
 
         API Documentation: https://www.linode.com/docs/api/object-storage/#object-storage-object-url-create
 
-        :param cluster_id: The ID of the cluster this bucket exists in.
-        :type cluster_id: str
+        :param cluster_or_region_id: The ID of the cluster or region this bucket exists in.
+        :type cluster_or_region_id: str
 
         :param bucket: The bucket name.
         :type bucket: str
@@ -337,7 +446,7 @@ def object_url_create(
 
         result = self.client.post(
             "/object-storage/buckets/{}/{}/object-url".format(
-                parse.quote(str(cluster_id)), parse.quote(str(bucket))
+                parse.quote(str(cluster_or_region_id)), parse.quote(str(bucket))
             ),
             data=drop_null_keys(params),
         )

linode_api4/objects/object_storage.py

@@ -1,6 +1,8 @@
 from typing import Optional
 from urllib import parse
 
+from deprecated import deprecated
+
 from linode_api4.errors import UnexpectedResponseError
 from linode_api4.objects import (
     Base,
@@ -21,19 +23,25 @@ class ObjectStorageACL(StrEnum):
     CUSTOM = "custom"
 
 
+class ObjectStorageKeyPermission(StrEnum):
+    READ_ONLY = "read_only"
+    READ_WRITE = "read_write"
+
+
 class ObjectStorageBucket(DerivedBase):
     """
     A bucket where objects are stored in.
 
     API documentation: https://www.linode.com/docs/api/object-storage/#object-storage-bucket-view
     """
 
-    api_endpoint = "/object-storage/buckets/{cluster}/{label}"
-    parent_id_name = "cluster"
+    api_endpoint = "/object-storage/buckets/{region}/{label}"
+    parent_id_name = "region"
     id_attribute = "label"
 
     properties = {
-        "cluster": Property(identifier=True),
+        "region": Property(identifier=True),
+        "cluster": Property(),
         "created": Property(is_datetime=True),
         "hostname": Property(),
         "label": Property(identifier=True),
@@ -57,8 +65,11 @@ def make_instance(cls, id, client, parent_id=None, json=None):
         """
         if json is None:
             return None
-        if parent_id is None and json["cluster"]:
-            parent_id = json["cluster"]
+
+        cluster_or_region = json.get("region") or json.get("cluster")
+
+        if parent_id is None and cluster_or_region:
+            parent_id = cluster_or_region
 
         if parent_id:
             return super().make(id, client, cls, parent_id=parent_id, json=json)
@@ -386,6 +397,13 @@ def object_acl_config_update(self, acl: ObjectStorageACL, name):
 
         return MappedObject(**result)
 
+    @deprecated(
+        reason=(
+            "'access' method has been deprecated in favor of the class method "
+            "'bucket_access' in ObjectStorageGroup, which can be accessed by "
+            "'client.object_storage.access'"
+        )
+    )
     def access(self, cluster, bucket_name, permissions):
         """
         Returns a dict formatted to be included in the `bucket_access` argument
@@ -411,8 +429,14 @@ def access(self, cluster, bucket_name, permissions):
         }
 
 
+@deprecated(
+    reason="deprecated to use regions list API for viewing available OJB clusters"
+)
 class ObjectStorageCluster(Base):
     """
+    This class will be deprecated to use the regions list to view available OBJ clusters,
+    and a new access key API will directly expose the S3 endpoint hostname.
+
     A cluster where Object Storage is available.
 
     API documentation: https://www.linode.com/docs/api/object-storage/#cluster-view
@@ -428,6 +452,13 @@ class ObjectStorageCluster(Base):
         "static_site_domain": Property(),
     }
 
+    @deprecated(
+        reason=(
+            "'buckets_in_cluster' method has been deprecated, please consider "
+            "switching to 'buckets_in_region' in the object storage group (can "
+            "be accessed via 'client.object_storage.buckets_in_cluster')."
+        )
+    )
     def buckets_in_cluster(self, *filters):
         """
         Returns a list of Buckets in this cluster belonging to this Account.
@@ -470,4 +501,5 @@ class ObjectStorageKeys(Base):
         "secret_key": Property(),
         "bucket_access": Property(),
         "limited": Property(),
+        "regions": Property(unordered=True),
     }