Refactor imports and update dependencies; remove unused AWS storage options methods

Author: legout

pyproject.toml

@@ -3,7 +3,7 @@ authors = [{name = "Volker L.", email = "[email protected]"}]
 dependencies = [
   "sf-hamilton[visualization]>=1.69.0",
   "python-dotenv>=1.0.1",
-  "fsspec>=2024.5.0",
+  "fsspec>=2024.10.0",
   "pyyaml>=6.0.1",
   "typer>=0.12.3",
   "rich>=13.9.3",
@@ -13,6 +13,7 @@ dependencies = [
   "pendulum>=3.0.0",
   "tqdm>=4.67.1",
   "pydantic>=2.10.2",
+  "s3fs>=2024.10.0",
 ]
 description = "A simple workflow framework. Hamilton + APScheduler = FlowerPower"
 keywords = [
@@ -111,5 +112,6 @@ dev-dependencies = [
   "asyncpg>=0.30.0",
   "paho-mqtt>=2.1.0",
   "greenlet>=3.1.1",
+  "obstore>=0.3.0",
 ]
 managed = true

src/flowerpower/io/base.py

@@ -94,49 +94,6 @@ def model_post_init(self, __context):
                 .rstrip("/")
             )
 
-    # def _update_storage_options_from_aws_credentials(
-    #     self,
-    #     profile: str = "default",
-    #     allow_invalid_certificates: bool = False,
-    #     allow_http: bool = False,
-    # ) -> AwsStorageOptions:
-    #     if isinstance(self.storage_options, AwsStorageOptions):
-    #         self.storage_options = self.storage_options.to_aws_credentials(
-    #             profile=profile,
-    #             allow_invalid_certificates=allow_invalid_certificates,
-    #             allow_http=allow_http,
-    #         )
-    #     self._update_fs()
-
-    # def _update_storage_options_from_env(
-    #     self,
-    # ):
-    #     protocol = get_protocol(self.path)
-    #     if protocol == "s3":
-    #         self.storage_options = AwsStorageOptions.from_env()
-    #     elif protocol == "gs" or protocol == "gcs":
-    #         self.storage_options = GcsStorageOptions.from_env()
-    #     elif protocol == "az" or protocol == "abfs":
-    #         self.storage_options = AzureStorageOptions.from_env()
-    #     elif protocol == "github":
-    #         self.storage_options = GitHubStorageOptions.from_env()
-    #     elif protocol == "gitlab":
-    #         self.storage_options = GitLabStorageOptions.from_env()
-    #     self._update_fs()
-
-    # def _update_storage_options(self, **kwargs):
-    #     self.storage_options = self.storage_options.model_copy(update=kwargs)
-    #     self._update_fs()
-
-    # def _update_fs(self):
-    #     if self.fs is None:
-    #         self.fs = get_filesystem(
-    #             path=self.path if isinstance(self.path, str) else self.path[0],
-    #             storage_options=self.storage_options,
-    #             fs=self.fs,
-    #             dirfs=False,
-    #         )
-
     @property
     def _path(self):
         if self.fs.protocol == "dir":

src/flowerpower/io/saver/deltatable.py

@@ -4,7 +4,7 @@
 import polars as pl
 import pandas as pd
 from typing import Any
-from ..utils.misc import _dict_to_dataframe
+from ...utils.misc import _dict_to_dataframe
 
 
 class DeltaTableWriter:

src/flowerpower/utils/filesystem/ext.py

@@ -880,7 +880,7 @@ def write_parquet(
     Returns:
         (pq.FileMetaData): Parquet file metadata.
     """
-    if isinstance(data, dict | list[dict]):
+    if isinstance(data, (dict | list)):
         data = _dict_to_dataframe(data)
 
     if isinstance(data, pl.LazyFrame):