Test Python 3.9 and 3.10 on CI; Apply fixes (#372)

* Expand Python versions on CI

* Guard typing imports

* Runtime fixes

* fix import

* fix get_ranges

* skip some tests on 3.9

Author: kylebarron

.github/workflows/test-python.yml

@@ -44,7 +44,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
 

obstore/python/obstore/_attributes.pyi

@@ -1,4 +1,10 @@
-from typing import Literal, TypeAlias
+import sys
+from typing import Literal
+
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
 
 Attribute: TypeAlias = (
     Literal[

obstore/python/obstore/_list.pyi

@@ -1,12 +1,18 @@
 # ruff: noqa: A001, UP006, UP035
 
+import sys
 from datetime import datetime
-from typing import Generic, List, Literal, Self, TypedDict, TypeVar, overload
+from typing import Generic, List, Literal, TypedDict, TypeVar, overload
 
 from arro3.core import RecordBatch, Table
 
 from ._store import ObjectStore
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 class ObjectMeta(TypedDict):
     """The metadata that describes an object."""
 

obstore/python/obstore/_put.pyi

@@ -1,11 +1,16 @@
 import sys
 from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator
 from pathlib import Path
-from typing import IO, Literal, TypeAlias, TypedDict
+from typing import IO, Literal, TypedDict
 
 from ._attributes import Attributes
 from .store import ObjectStore
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
 if sys.version_info >= (3, 12):
     from collections.abc import Buffer
 else:

obstore/python/obstore/_sign.pyi

@@ -1,9 +1,15 @@
+import sys
 from collections.abc import Sequence
 from datetime import timedelta
-from typing import Literal, TypeAlias, overload
+from typing import Literal, overload
 
 from .store import AzureStore, GCSStore, S3Store
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
 HTTP_METHOD: TypeAlias = Literal[
     "GET",
     "PUT",

obstore/python/obstore/_store/__init__.pyi

@@ -1,7 +1,8 @@
 # TODO: move to reusable types package
+import sys
 from collections.abc import Callable
 from pathlib import Path
-from typing import Any, Self, TypeAlias, Unpack, overload
+from typing import Any, overload
 
 from ._aws import S3Config as S3Config
 from ._aws import S3Credential as S3Credential
@@ -23,6 +24,16 @@ from ._http import HTTPStore as HTTPStore
 from ._retry import BackoffConfig as BackoffConfig
 from ._retry import RetryConfig as RetryConfig
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
+if sys.version_info >= (3, 11):
+    from typing import Self, Unpack
+else:
+    from typing_extensions import Self, Unpack
+
 @overload
 def from_url(
     url: str,

obstore/python/obstore/_store/_aws.pyi

@@ -1,19 +1,21 @@
+import sys
 from collections.abc import Coroutine
 from datetime import datetime
-from typing import (
-    Any,
-    Literal,
-    NotRequired,
-    Protocol,
-    Self,
-    TypeAlias,
-    TypedDict,
-    Unpack,
-)
+from typing import Any, Literal, Protocol, TypedDict
 
 from ._client import ClientConfig
 from ._retry import RetryConfig
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
+if sys.version_info >= (3, 11):
+    from typing import NotRequired, Self, Unpack
+else:
+    from typing_extensions import NotRequired, Self, Unpack
+
 # To update s3 region list:
 # import pandas as pd  # noqa: ERA001
 # url = "https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html"  # noqa: ERA001

obstore/python/obstore/_store/_azure.pyi

@@ -1,10 +1,21 @@
+import sys
 from collections.abc import Coroutine
 from datetime import datetime
-from typing import Any, Protocol, Self, TypeAlias, TypedDict, Unpack
+from typing import Any, Protocol, TypedDict
 
 from ._client import ClientConfig
 from ._retry import RetryConfig
 
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
+
+if sys.version_info >= (3, 11):
+    from typing import Self, Unpack
+else:
+    from typing_extensions import Self, Unpack
+
 class AzureConfig(TypedDict, total=False):
     """Configuration parameters for AzureStore.
 

obstore/python/obstore/_store/_gcs.pyi

@@ -1,10 +1,16 @@
+import sys
 from collections.abc import Coroutine
 from datetime import datetime
-from typing import Any, Protocol, Self, TypedDict, Unpack
+from typing import Any, Protocol, TypedDict
 
 from ._client import ClientConfig
 from ._retry import RetryConfig
 
+if sys.version_info >= (3, 11):
+    from typing import Self, Unpack
+else:
+    from typing_extensions import Self, Unpack
+
 class GCSConfig(TypedDict, total=False):
     """Configuration parameters for GCSStore.
 

obstore/python/obstore/_store/_http.pyi

@@ -1,8 +1,13 @@
-from typing import Self
+import sys
 
 from ._client import ClientConfig
 from ._retry import RetryConfig
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
 class HTTPStore:
     """Configure a connection to a generic HTTP server."""
 

obstore/python/obstore/auth/boto3.py

@@ -2,20 +2,26 @@
 
 from __future__ import annotations
 
-from datetime import UTC, datetime, timedelta
-from typing import TYPE_CHECKING, NotRequired, TypedDict, Unpack
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, TypedDict
 
 import boto3
 import boto3.session
 import botocore.credentials
 
 if TYPE_CHECKING:
+    import sys
     from collections.abc import Sequence
 
     import botocore.session
 
     from obstore.store import S3Config, S3Credential
 
+    if sys.version_info >= (3, 11):
+        from typing import NotRequired, Unpack
+    else:
+        from typing_extensions import NotRequired, Unpack
+
 
 class PolicyDescriptorTypeTypeDef(TypedDict):  # noqa: D101
     arn: NotRequired[str]
@@ -97,7 +103,7 @@ def __init__(
 
     def __call__(self) -> S3Credential:
         """Fetch credentials."""
-        expires_at = datetime.now(UTC) + self.ttl
+        expires_at = datetime.now(timezone.utc) + self.ttl
         frozen_credentials = self.credentials.get_frozen_credentials()
         return {
             "access_key_id": frozen_credentials.access_key,

obstore/python/obstore/auth/google.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, cast
 
 import google.auth
@@ -156,4 +156,4 @@ def _replace_expiry_timezone_utc(expiry: datetime | None) -> datetime | None:
     if expiry is None:
         return None
 
-    return expiry.replace(tzinfo=UTC) if expiry.tzinfo is None else expiry
+    return expiry.replace(tzinfo=timezone.utc) if expiry.tzinfo is None else expiry

obstore/python/obstore/fsspec.py

@@ -37,7 +37,7 @@
 from collections import defaultdict
 from functools import lru_cache
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, Unpack, overload
+from typing import TYPE_CHECKING, Literal, overload
 from urllib.parse import urlparse
 
 import fsspec.asyn
@@ -48,7 +48,9 @@
 from obstore.store import from_url
 
 if TYPE_CHECKING:
+    import sys
     from collections.abc import Coroutine, Iterable
+    from typing import Any
 
     from obstore import Attributes, Bytes, ReadableFile, WritableFile
     from obstore.store import (
@@ -60,6 +62,12 @@
         S3Config,
     )
 
+    if sys.version_info >= (3, 11):
+        from typing import Unpack
+    else:
+        from typing_extensions import Unpack
+
+
 __all__ = [
     "BufferedFile",
     "FsspecStore",
@@ -391,9 +399,8 @@ async def _cat_ranges(  # noqa: PLR0913
             raise ValueError
 
         per_file_requests: dict[str, list[tuple[int, int, int]]] = defaultdict(list)
-        for idx, (path, start, end) in enumerate(
-            zip(paths, starts, ends, strict=False),
-        ):
+        # When upgrading to Python 3.10, use strict=True
+        for idx, (path, start, end) in enumerate(zip(paths, starts, ends)):
             per_file_requests[path].append((start, end, idx))
 
         futs: list[Coroutine[Any, Any, list[Bytes]]] = []
@@ -409,13 +416,11 @@ async def _cat_ranges(  # noqa: PLR0913
         result = await asyncio.gather(*futs)
 
         output_buffers: list[bytes] = [b""] * len(paths)
-        for per_file_request, buffers in zip(
-            per_file_requests.items(),
-            result,
-            strict=True,
-        ):
+        # When upgrading to Python 3.10, use strict=True
+        for per_file_request, buffers in zip(per_file_requests.items(), result):
             path, ranges = per_file_request
-            for buffer, ranges_ in zip(buffers, ranges, strict=True):
+            # When upgrading to Python 3.10, use strict=True
+            for buffer, ranges_ in zip(buffers, ranges):
                 initial_index = ranges_[2]
                 output_buffers[initial_index] = buffer.to_bytes()
 

obstore/python/obstore/store.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Literal, TypeAlias, Unpack, overload
+from typing import TYPE_CHECKING, Union, overload
 
 import obstore as obs
 from obstore._obstore import _store
@@ -22,7 +22,7 @@
         Sequence,
     )
     from pathlib import Path
-    from typing import IO, Literal
+    from typing import IO, Any, Literal
 
     from arro3.core import RecordBatch, Table
 
@@ -54,6 +54,16 @@
         S3CredentialProvider,
     )
 
+    if sys.version_info >= (3, 10):
+        from typing import TypeAlias
+    else:
+        from typing_extensions import TypeAlias
+
+    if sys.version_info >= (3, 11):
+        from typing import Unpack
+    else:
+        from typing_extensions import Unpack
+
     if sys.version_info >= (3, 12):
         from collections.abc import Buffer
     else:
@@ -579,9 +589,14 @@ class S3Store(_ObjectStoreMixin, _store.S3Store):
     """
 
 
-ObjectStore: TypeAlias = (
-    AzureStore | GCSStore | HTTPStore | S3Store | LocalStore | MemoryStore
-)
+ObjectStore: TypeAlias = Union[
+    AzureStore,
+    GCSStore,
+    HTTPStore,
+    S3Store,
+    LocalStore,
+    MemoryStore,
+]
 """All supported ObjectStore implementations."""