diff --git a/docs/nav/reference/storage-gcs.md b/docs/nav/reference/storage-gcs.md new file mode 100644 index 00000000..6ee437f7 --- /dev/null +++ b/docs/nav/reference/storage-gcs.md @@ -0,0 +1,3 @@ +# StorageGCS + +::: dotflow.providers.storage_gcs.StorageGCS diff --git a/docs/nav/tutorial/storage-gcs.md b/docs/nav/tutorial/storage-gcs.md new file mode 100644 index 00000000..e4db8bb4 --- /dev/null +++ b/docs/nav/tutorial/storage-gcs.md @@ -0,0 +1,25 @@ +# Storage GCS + +Persists task output to Google Cloud Storage. Good for serverless and cloud-native workflows on GCP. + +/// note +Requires `pip install dotflow[gcp]` +/// + +## Example + +{* ./docs_src/storage/storage_gcs.py ln[1:34] hl[2,16:22] *} + +## Authentication + +`StorageGCS` uses Application Default Credentials (ADC): + +1. Environment variable: `GOOGLE_APPLICATION_CREDENTIALS` pointing to a service account JSON +2. `gcloud auth application-default login` for local development +3. Service account: automatic on Cloud Run, Cloud Functions, GKE + +No credentials are needed in code — the GCP client handles it transparently. + +## References + +- [StorageGCS](https://dotflow-io.github.io/dotflow/nav/reference/storage-gcs/) diff --git a/docs_src/storage/storage_gcs.py b/docs_src/storage/storage_gcs.py new file mode 100644 index 00000000..3a91b1b7 --- /dev/null +++ b/docs_src/storage/storage_gcs.py @@ -0,0 +1,36 @@ +from dotflow import Config, DotFlow, action +from dotflow.providers import StorageGCS + + +@action +def step_one(): + return {"message": "hello from GCS"} + + +@action +def step_two(previous_context): + print(previous_context.storage) + return "ok" + + +config = Config( + storage=StorageGCS( + bucket="dotflow-io-bucket", + prefix="workflows/", + project="etl-test", + ) +) + + +def main(): + workflow = DotFlow(config=config) + + workflow.task.add(step=step_one) + workflow.task.add(step=step_two) + workflow.start() + + return workflow + + +if __name__ == "__main__": + main() diff --git a/dotflow/providers/__init__.py b/dotflow/providers/__init__.py index 9dee74d4..d46a2584 100644 --- a/dotflow/providers/__init__.py +++ b/dotflow/providers/__init__.py @@ -13,6 +13,7 @@ "StorageDefault", "StorageFile", "StorageS3", + "StorageGCS", ] @@ -21,4 +22,10 @@ def __getattr__(name): from dotflow.providers.storage_s3 import StorageS3 return StorageS3 + + if name == "StorageGCS": + from dotflow.providers.storage_gcs import StorageGCS + + return StorageGCS + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/dotflow/providers/storage_gcs.py b/dotflow/providers/storage_gcs.py new file mode 100644 index 00000000..e39b1bf6 --- /dev/null +++ b/dotflow/providers/storage_gcs.py @@ -0,0 +1,119 @@ +"""Storage GCS""" + +from collections.abc import Callable +from json import dumps, loads +from typing import Any + +from dotflow.abc.storage import Storage +from dotflow.core.context import Context +from dotflow.core.exception import ModuleNotFound + + +class StorageGCS(Storage): + """ + Import: + You can import the **StorageGCS** class directly from dotflow providers: + + from dotflow.providers import StorageGCS + + Example: + `class` dotflow.providers.storage_gcs.StorageGCS + + from dotflow import Config + from dotflow.providers import StorageGCS + + config = Config( + storage=StorageGCS( + bucket="my-dotflow-bucket", + prefix="workflows/", + project="my-gcp-project" + ) + ) + + Args: + bucket (str): GCS bucket name. + + prefix (str): Key prefix for all stored objects. + + project (str): GCP project ID. Defaults to ADC project. + """ + + def __init__( + self, + *args, + bucket: str, + prefix: str = "dotflow/", + project: str = None, + **kwargs, + ): + try: + from google.api_core.exceptions import NotFound + from google.cloud import storage as gcs + except ImportError: + raise ModuleNotFound( + module="google-cloud-storage", + library="dotflow[gcp]", + ) from None + + self._not_found = NotFound + self.client = gcs.Client(project=project) + self.bucket_obj = self.client.bucket(bucket) + self.bucket_obj.reload() + self.prefix = prefix + + def post(self, key: str, context: Context) -> None: + task_context = [] + + if isinstance(context.storage, list): + for item in context.storage: + if isinstance(item, Context): + task_context.append(self._dumps(storage=item.storage)) + else: + task_context.append(self._dumps(storage=context.storage)) + + self._write(key=key, data=task_context) + + def get(self, key: str) -> Context: + task_context = self._read(key) + + if len(task_context) == 0: + return Context() + + if len(task_context) == 1: + return self._loads(storage=task_context[0]) + + contexts = Context(storage=[]) + for context in task_context: + contexts.storage.append(self._loads(storage=context)) + + return contexts + + def key(self, task: Callable): + return f"{task.workflow_id}-{task.task_id}" + + def _read(self, key: str) -> list: + blob = self.bucket_obj.blob(f"{self.prefix}{key}") + try: + data = blob.download_as_text() + return loads(data) + except self._not_found: + return [] + + def _write(self, key: str, data: list) -> None: + blob = self.bucket_obj.blob(f"{self.prefix}{key}") + blob.upload_from_string( + dumps(data), + content_type="application/json", + ) + + def _loads(self, storage: Any) -> Context: + try: + return Context(storage=loads(storage)) + except Exception: + return Context(storage=storage) + + def _dumps(self, storage: Any) -> str: + try: + return dumps(storage) + except TypeError: + return str(storage) diff --git a/mkdocs.yml b/mkdocs.yml index ec58b48a..86788bb2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -130,6 +130,7 @@ nav: - nav/tutorial/storage-default.md - nav/tutorial/storage-file.md - nav/tutorial/storage-s3.md + - nav/tutorial/storage-gcs.md - nav/tutorial/provider-notify.md - nav/tutorial/provider-log.md - Tutorial - User Guide: @@ -187,6 +188,7 @@ nav: - nav/reference/storage-init.md - nav/reference/storage-file.md - nav/reference/storage-s3.md + - nav/reference/storage-gcs.md - Instance: - nav/reference/task-instance.md - nav/reference/context-instance.md diff --git a/poetry.lock b/poetry.lock index 6bcc3ff3..1de50c54 100644 --- a/poetry.lock +++ b/poetry.lock @@ -148,7 +148,7 @@ description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "cachetools-7.0.5-py3-none-any.whl", hash = "sha256:46bc8ebefbe485407621d0a4264b23c080cedd913921bad7ac3ed2f26c183114"}, {file = "cachetools-7.0.5.tar.gz", hash = "sha256:0cd042c24377200c1dcd225f8b7b12b0ca53cc2c961b43757e774ebe190fd990"}, @@ -172,8 +172,7 @@ version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.9" -groups = ["dev"] -markers = "implementation_name == \"pypy\" or platform_python_implementation != \"PyPy\"" +groups = ["main", "dev"] files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -260,6 +259,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] +markers = {main = "extra == \"gcp\" and platform_python_implementation != \"PyPy\"", dev = "platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -439,7 +439,7 @@ description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" groups = ["docs"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d"}, {file = "click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5"}, @@ -588,7 +588,7 @@ description = "Code coverage measurement for Python" optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "coverage-7.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0723d2c96324561b9aa76fb982406e11d93cdb388a7a7da2b16e04719cf7ca5"}, {file = "coverage-7.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52f444e86475992506b32d4e5ca55c24fc88d73bcbda0e9745095b28ef4dc0cf"}, @@ -710,8 +710,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" -groups = ["dev"] -markers = "python_version < \"3.10\"" +groups = ["main", "dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -741,6 +740,7 @@ files = [ {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] +markers = {main = "python_version < \"3.10\" and extra == \"gcp\"", dev = "python_version < \"3.10\""} [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} @@ -761,8 +761,7 @@ version = "46.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" -groups = ["dev"] -markers = "python_version >= \"3.10\"" +groups = ["main", "dev"] files = [ {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, @@ -814,6 +813,7 @@ files = [ {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, ] +markers = {main = "extra == \"gcp\" and python_version >= \"3.10\"", dev = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\""} [package.dependencies] cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} @@ -898,7 +898,7 @@ description = "DNS toolkit" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "python_version >= \"3.10\" and extra == \"mongodb\"" +markers = "extra == \"mongodb\" and python_version >= \"3.10\"" files = [ {file = "dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af"}, {file = "dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f"}, @@ -969,7 +969,7 @@ description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70"}, {file = "filelock-3.25.2.tar.gz", hash = "sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694"}, @@ -1010,6 +1010,219 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "google-api-core" +version = "2.30.2" +description = "Google API client core library" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "google_api_core-2.30.2-py3-none-any.whl", hash = "sha256:a4c226766d6af2580577db1f1a51bf53cd262f722b49731ce7414c43068a9594"}, + {file = "google_api_core-2.30.2.tar.gz", hash = "sha256:9a8113e1a88bdc09a7ff629707f2214d98d61c7f6ceb0ea38c42a095d02dc0f9"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.63.2,<2.0.0" +proto-plus = [ + {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, +] +protobuf = ">=4.25.8,<8.0.0" +requests = ">=2.20.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] + +[[package]] +name = "google-auth" +version = "2.49.1" +description = "Google Authentication Library" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "google_auth-2.49.1-py3-none-any.whl", hash = "sha256:195ebe3dca18eddd1b3db5edc5189b76c13e96f29e73043b923ebcf3f1a860f7"}, + {file = "google_auth-2.49.1.tar.gz", hash = "sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64"}, +] + +[package.dependencies] +cryptography = ">=38.0.3" +pyasn1-modules = ">=0.2.1" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] +cryptography = ["cryptography (>=38.0.3)"] +enterprise-cert = ["pyopenssl"] +pyjwt = ["pyjwt (>=2.0)"] +pyopenssl = ["pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0)"] +rsa = ["rsa (>=3.1.4,<5)"] +testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "flask", "freezegun", "grpcio", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] +urllib3 = ["packaging", "urllib3"] + +[[package]] +name = "google-cloud-core" +version = "2.5.1" +description = "Google Cloud API client core library" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "google_cloud_core-2.5.1-py3-none-any.whl", hash = "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7"}, + {file = "google_cloud_core-2.5.1.tar.gz", hash = "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811"}, +] + +[package.dependencies] +google-api-core = ">=2.11.0,<3.0.0" +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.38.0,<2.0.0)"] + +[[package]] +name = "google-cloud-storage" +version = "3.9.0" +description = "Google Cloud Storage API client library" +optional = true +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.10\" and extra == \"gcp\"" +files = [ + {file = "google_cloud_storage-3.9.0-py3-none-any.whl", hash = "sha256:2dce75a9e8b3387078cbbdad44757d410ecdb916101f8ba308abf202b6968066"}, + {file = "google_cloud_storage-3.9.0.tar.gz", hash = "sha256:f2d8ca7db2f652be757e92573b2196e10fbc09649b5c016f8b422ad593c641cc"}, +] + +[package.dependencies] +google-api-core = ">=2.27.0,<3.0.0" +google-auth = ">=2.26.1,<3.0.0" +google-cloud-core = ">=2.4.2,<3.0.0" +google-crc32c = ">=1.1.3,<2.0.0" +google-resumable-media = ">=2.7.2,<3.0.0" +requests = ">=2.22.0,<3.0.0" + +[package.extras] +grpc = ["google-api-core[grpc] (>=2.27.0,<3.0.0)", "grpc-google-iam-v1 (>=0.14.0,<1.0.0)", "grpcio (>=1.33.2,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.76.0,<2.0.0)", "proto-plus (>=1.22.3,<2.0.0) ; python_version < \"3.13\"", "proto-plus (>=1.25.0,<2.0.0) ; python_version >= \"3.13\"", "protobuf (>=3.20.2,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<7.0.0)"] +protobuf = ["protobuf (>=3.20.2,<7.0.0)"] +testing = ["PyYAML", "black", "brotli", "coverage", "flake8", "google-cloud-iam", "google-cloud-kms", "google-cloud-pubsub", "google-cloud-testutils", "google-cloud-testutils", "mock", "numpy", "opentelemetry-sdk", "psutil", "py-cpuinfo", "pyopenssl", "pytest", "pytest-asyncio", "pytest-benchmark", "pytest-cov", "pytest-rerunfailures", "pytest-xdist"] +tracing = ["opentelemetry-api (>=1.1.0,<2.0.0)"] + +[[package]] +name = "google-cloud-storage" +version = "3.10.1" +description = "Google Cloud Storage API client library" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"gcp\" and python_version >= \"3.10\"" +files = [ + {file = "google_cloud_storage-3.10.1-py3-none-any.whl", hash = "sha256:a72f656759b7b99bda700f901adcb3425a828d4a29f911bc26b3ea79c5b1217f"}, + {file = "google_cloud_storage-3.10.1.tar.gz", hash = "sha256:97db9aa4460727982040edd2bd13ff3d5e2260b5331ad22895802da1fc2a5286"}, +] + +[package.dependencies] +google-api-core = ">=2.27.0,<3.0.0" +google-auth = ">=2.26.1,<3.0.0" +google-cloud-core = ">=2.4.2,<3.0.0" +google-crc32c = ">=1.1.3,<2.0.0" +google-resumable-media = ">=2.7.2,<3.0.0" +requests = ">=2.22.0,<3.0.0" + +[package.extras] +grpc = ["google-api-core[grpc] (>=2.27.0,<3.0.0)", "grpc-google-iam-v1 (>=0.14.0,<1.0.0)", "grpcio (>=1.33.2,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.76.0,<2.0.0)", "proto-plus (>=1.22.3,<2.0.0) ; python_version < \"3.13\"", "proto-plus (>=1.25.0,<2.0.0) ; python_version >= \"3.13\"", "protobuf (>=3.20.2,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<7.0.0)"] +protobuf = ["protobuf (>=3.20.2,<7.0.0)"] +testing = ["PyYAML", "black", "brotli", "coverage", "flake8", "google-cloud-iam", "google-cloud-kms", "google-cloud-pubsub", "google-cloud-testutils", "google-cloud-testutils", "mock", "numpy", "opentelemetry-sdk", "psutil", "py-cpuinfo", "pyopenssl", "pytest", "pytest-asyncio", "pytest-benchmark", "pytest-cov", "pytest-rerunfailures", "pytest-xdist"] +tracing = ["opentelemetry-api (>=1.1.0,<2.0.0)"] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "google_crc32c-1.8.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff"}, + {file = "google_crc32c-1.8.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288"}, + {file = "google_crc32c-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d"}, + {file = "google_crc32c-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092"}, + {file = "google_crc32c-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733"}, + {file = "google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8"}, + {file = "google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7"}, + {file = "google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15"}, + {file = "google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a"}, + {file = "google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2"}, + {file = "google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113"}, + {file = "google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb"}, + {file = "google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411"}, + {file = "google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454"}, + {file = "google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962"}, + {file = "google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b"}, + {file = "google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27"}, + {file = "google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa"}, + {file = "google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8"}, + {file = "google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f"}, + {file = "google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697"}, + {file = "google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651"}, + {file = "google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2"}, + {file = "google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21"}, + {file = "google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2"}, + {file = "google_crc32c-1.8.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc"}, + {file = "google_crc32c-1.8.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2"}, + {file = "google_crc32c-1.8.0-cp39-cp39-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215"}, + {file = "google_crc32c-1.8.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a"}, + {file = "google_crc32c-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f"}, + {file = "google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93"}, + {file = "google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c"}, + {file = "google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79"}, +] + +[[package]] +name = "google-resumable-media" +version = "2.8.2" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "google_resumable_media-2.8.2-py3-none-any.whl", hash = "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220"}, + {file = "google_resumable_media-2.8.2.tar.gz", hash = "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70"}, +] + +[package.dependencies] +google-crc32c = ">=1.0.0,<2.0.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "google-auth (>=1.22.0,<2.0.0)"] +requests = ["requests (>=2.18.0,<3.0.0)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.74.0" +description = "Common protobufs used in Google APIs" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5"}, + {file = "googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1"}, +] + +[package.dependencies] +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0)"] + [[package]] name = "griffe" version = "1.13.0" @@ -1150,7 +1363,7 @@ description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, @@ -1333,7 +1546,7 @@ description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.10" groups = ["docs"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36"}, {file = "markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950"}, @@ -1391,7 +1604,7 @@ description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" groups = ["main"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, @@ -1890,7 +2103,7 @@ description = "Optional static typing for Python" optional = false python-versions = ">=3.10" groups = ["code-quality"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "mypy-1.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d99f515f95fd03a90875fdb2cca12ff074aa04490db4d190905851bdf8a549a8"}, {file = "mypy-1.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd0212976dc57a5bfeede7c219e7cd66568a32c05c9129686dd487c059c1b88a"}, @@ -2036,7 +2249,7 @@ description = "A small Python package for determining appropriate platform-speci optional = false python-versions = ">=3.10" groups = ["dev", "docs"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868"}, {file = "platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934"}, @@ -2058,6 +2271,94 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "proto-plus" +version = "1.27.2" +description = "Beautiful, Pythonic protocol buffers" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718"}, + {file = "proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24"}, +] + +[package.dependencies] +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "6.33.6" +description = "" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version < \"3.10\" and extra == \"gcp\"" +files = [ + {file = "protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3"}, + {file = "protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326"}, + {file = "protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593"}, + {file = "protobuf-6.33.6-cp39-cp39-win32.whl", hash = "sha256:bd56799fb262994b2c2faa1799693c95cc2e22c62f56fb43af311cae45d26f0e"}, + {file = "protobuf-6.33.6-cp39-cp39-win_amd64.whl", hash = "sha256:f443a394af5ed23672bc6c486be138628fbe5c651ccbc536873d7da23d1868cf"}, + {file = "protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901"}, + {file = "protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135"}, +] + +[[package]] +name = "protobuf" +version = "7.34.1" +description = "" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"gcp\" and python_version >= \"3.10\"" +files = [ + {file = "protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7"}, + {file = "protobuf-7.34.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b"}, + {file = "protobuf-7.34.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a"}, + {file = "protobuf-7.34.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4"}, + {file = "protobuf-7.34.1-cp310-abi3-win32.whl", hash = "sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a"}, + {file = "protobuf-7.34.1-cp310-abi3-win_amd64.whl", hash = "sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c"}, + {file = "protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11"}, + {file = "protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280"}, +] + +[[package]] +name = "pyasn1" +version = "0.6.3" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, + {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +description = "A collection of ASN.1-based protocols modules" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"gcp\"" +files = [ + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, +] + +[package.dependencies] +pyasn1 = ">=0.6.1,<0.7.0" + [[package]] name = "pycodestyle" version = "2.14.0" @@ -2076,12 +2377,12 @@ version = "2.23" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["dev"] -markers = "(implementation_name == \"pypy\" or platform_python_implementation != \"PyPy\") and implementation_name != \"PyPy\" and python_version < \"3.10\"" +groups = ["main", "dev"] files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] +markers = {main = "extra == \"gcp\" and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and python_version < \"3.10\"", dev = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and python_version < \"3.10\" and implementation_name != \"PyPy\""} [[package]] name = "pycparser" @@ -2089,12 +2390,12 @@ version = "3.0" description = "C parser in Python" optional = false python-versions = ">=3.10" -groups = ["dev"] -markers = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and python_version >= \"3.10\" and implementation_name != \"PyPy\"" +groups = ["main", "dev"] files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] +markers = {main = "platform_python_implementation != \"PyPy\" and extra == \"gcp\" and implementation_name != \"PyPy\" and python_version >= \"3.10\"", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and python_version >= \"3.10\" or implementation_name == \"pypy\" and python_version >= \"3.10\""} [[package]] name = "pydantic" @@ -2467,7 +2768,7 @@ description = "API to interact with the python pyproject.toml based projects" optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "pyproject_api-1.10.0-py3-none-any.whl", hash = "sha256:8757c41a79c0f4ab71b99abed52b97ecf66bd20b04fa59da43b5840bac105a09"}, {file = "pyproject_api-1.10.0.tar.gz", hash = "sha256:40c6f2d82eebdc4afee61c773ed208c04c19db4c4a60d97f8d7be3ebc0bbb330"}, @@ -2818,7 +3119,7 @@ description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a"}, {file = "requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517"}, @@ -2996,7 +3297,7 @@ description = "A lil' TOML writer" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90"}, {file = "tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"}, @@ -3047,7 +3348,7 @@ description = "tox is a generic virtualenv management and test command line tool optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "tox-4.52.0-py3-none-any.whl", hash = "sha256:624d8ea4a8c6d5e8d168eedf0e318d736fb22e83ca83137d001ac65ffdec46fd"}, {file = "tox-4.52.0.tar.gz", hash = "sha256:6054abf5c8b61d58776fbec991f9bf0d34bb883862beb93d2fe55601ef3977c9"}, @@ -3123,7 +3424,7 @@ description = "HTTP library with thread-safe connection pooling, file post, and optional = false python-versions = ">=3.9" groups = ["main", "dev", "docs"] -markers = "python_version >= \"3.10\"" +markers = "python_version >= \"3.10\" and python_version < \"3.12\" or python_version >= \"3.12\"" files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, @@ -3271,9 +3572,10 @@ type = ["pytest-mypy"] [extras] aws = ["boto3"] +gcp = ["google-cloud-storage"] mongodb = ["dotflow-mongodb"] [metadata] lock-version = "2.1" python-versions = ">=3.9.0" -content-hash = "118e109fd3d1eeed79b34beda12069d1db1f8c28d5e14e48105ba0b618f5e659" +content-hash = "766cc75e4df56de2ede225825d0feb8eb93bbe125bc3d1f764f48ce6682cab17" diff --git a/pyproject.toml b/pyproject.toml index 467d90b8..4bd03bda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ Documentation = "https://github.com/dotflow-io/dotflow/blob/master/README.md" [project.optional-dependencies] mongodb = ["dotflow-mongodb"] aws = ["boto3"] +gcp = ["google-cloud-storage"] [tool.poetry] name = "dotflow" diff --git a/tests/providers/test_storage_gcs.py b/tests/providers/test_storage_gcs.py new file mode 100644 index 00000000..1cb899db --- /dev/null +++ b/tests/providers/test_storage_gcs.py @@ -0,0 +1,137 @@ +"""Test StorageGCS""" + +import unittest +from json import dumps, loads +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +from dotflow.core.context import Context +from dotflow.core.task import Task +from tests.mocks import action_step + +BUCKET = "test-dotflow-bucket" +PREFIX = "dotflow/" +PROJECT = "test-project" + + +class TestStorageGCS(unittest.TestCase): + def setUp(self): + self.mock_client = MagicMock() + self.mock_bucket = MagicMock() + self.mock_client.bucket.return_value = self.mock_bucket + + patcher = patch( + "dotflow.providers.storage_gcs.StorageGCS.__init__", + return_value=None, + ) + patcher.start() + self.addCleanup(patcher.stop) + + from dotflow.providers.storage_gcs import StorageGCS + + self.storage = StorageGCS() + self.storage.client = self.mock_client + self.storage.bucket_obj = self.mock_bucket + self.storage.prefix = PREFIX + self.storage._not_found = Exception + + def test_storage_gcs_instance(self): + self.assertEqual(self.storage.prefix, PREFIX) + + def test_post(self): + expected_value = {"foo": "bar"} + mock_blob = MagicMock() + self.mock_bucket.blob.return_value = mock_blob + + self.storage.post(key="test", context=Context(storage=expected_value)) + + self.mock_bucket.blob.assert_called_once_with(f"{PREFIX}test") + mock_blob.upload_from_string.assert_called_once() + uploaded = mock_blob.upload_from_string.call_args + data = loads(uploaded[0][0]) + self.assertEqual(loads(data[0]), expected_value) + + def test_post_many(self): + expected_one = {"foo": True} + expected_two = {"foo": False} + mock_blob = MagicMock() + self.mock_bucket.blob.return_value = mock_blob + + input_value = Context( + storage=[ + Context(storage=expected_one), + Context(storage=expected_two), + ] + ) + + self.storage.post(key="test", context=input_value) + + uploaded = mock_blob.upload_from_string.call_args + data = loads(uploaded[0][0]) + self.assertEqual(loads(data[0]), expected_one) + self.assertEqual(loads(data[1]), expected_two) + + def test_post_overwrites_existing_key(self): + new_value = {"foo": "baz"} + mock_blob = MagicMock() + self.mock_bucket.blob.return_value = mock_blob + + self.storage.post(key="test", context=Context(storage=new_value)) + + uploaded = mock_blob.upload_from_string.call_args + data = loads(uploaded[0][0]) + self.assertEqual(len(data), 1) + self.assertEqual(loads(data[0]), new_value) + + def test_get(self): + expected_value = {"foo": "bar"} + mock_blob = MagicMock() + mock_blob.exists.return_value = True + mock_blob.download_as_text.return_value = dumps( + [dumps(expected_value)] + ) + self.mock_bucket.blob.return_value = mock_blob + + result = self.storage.get(key="test") + + self.assertIsInstance(result, Context) + self.assertEqual(result.storage, expected_value) + + def test_get_many(self): + expected_one = {"foo": "bar"} + expected_two = True + mock_blob = MagicMock() + mock_blob.exists.return_value = True + mock_blob.download_as_text.return_value = dumps( + [dumps(expected_one), dumps(expected_two)] + ) + self.mock_bucket.blob.return_value = mock_blob + + result = self.storage.get(key="test") + + self.assertIsInstance(result, Context) + self.assertEqual(result.storage[0].storage, expected_one) + self.assertEqual(result.storage[1].storage, expected_two) + + def test_get_nonexistent_key(self): + mock_blob = MagicMock() + mock_blob.download_as_text.side_effect = Exception("NotFound") + self.mock_bucket.blob.return_value = mock_blob + + result = self.storage.get(key="nonexistent") + + self.assertIsInstance(result, Context) + self.assertIsNone(result.storage) + + def test_key(self): + workflow_id = uuid4() + + task = Task( + task_id=0, + workflow_id=workflow_id, + step=action_step, + ) + + result = self.storage.key(task=task) + + self.assertEqual(result, f"{workflow_id}-0")