Add force_path_style S3 config to force path-style addressing (#805)

* Add `force_path_style` S3 config to force path-style addressing

Closes #793

* Cleanup

* Switch to bool

* Add serde default

* fix

* Fix more tests

* fix one more test

* fix one last time.

* couple more.

* last one? !

Author: dcherian

docs/docs/icechunk-python/storage.md

@@ -105,6 +105,7 @@ icechunk.s3_storage(
     secret_access_key='minio123',
     endpoint_url='http://localhost:9000',
     allow_http=True,
+    force_path_style=True,
 ```
 
 A few things to note:

icechunk-python/python/icechunk/_icechunk_python.pyi

@@ -12,6 +12,7 @@ class S3Options:
         endpoint_url: str | None = None,
         allow_http: bool = False,
         anonymous: bool = False,
+        force_path_style: bool = False,
     ) -> None:
         """
         Create a new `S3Options` object
@@ -26,6 +27,8 @@ class S3Options:
             Whether to allow HTTP requests to the storage backend.
         anonymous: bool
             Whether to use anonymous credentials to the storage backend. When `True`, the s3 requests will not be signed.
+        force_path_style: bool
+            Whether to force use of path-style addressing for buckets.
         """
 
 class ObjectStoreConfig:

icechunk-python/python/icechunk/credentials.py

@@ -323,6 +323,7 @@ def containers_credentials(
         endpoint_url="http://localhost:9000",
         allow_http=True,
         s3_compatible=True,
+        force_path_style=True,
     )
     container = ic.VirtualChunkContainer("s3", "s3://", virtual_store_config)
     config.set_virtual_chunk_container(container)

icechunk-python/python/icechunk/storage.py

@@ -46,9 +46,15 @@ def s3_store(
     allow_http: bool = False,
     anonymous: bool = False,
     s3_compatible: bool = False,
+    force_path_style: bool = False,
 ) -> ObjectStoreConfig.S3Compatible | ObjectStoreConfig.S3:
     """Build an ObjectStoreConfig instance for S3 or S3 compatible object stores."""
-    options = S3Options(region=region, endpoint_url=endpoint_url, allow_http=allow_http)
+    options = S3Options(
+        region=region,
+        endpoint_url=endpoint_url,
+        allow_http=allow_http,
+        force_path_style=force_path_style,
+    )
     return (
         ObjectStoreConfig.S3Compatible(options)
         if s3_compatible
@@ -70,6 +76,7 @@ def s3_storage(
     anonymous: bool | None = None,
     from_env: bool | None = None,
     get_credentials: Callable[[], S3StaticCredentials] | None = None,
+    force_path_style: bool = False,
 ) -> Storage:
     """Create a Storage instance that saves data in S3 or S3 compatible object stores.
 
@@ -99,6 +106,8 @@ def s3_storage(
         Fetch credentials from the operative system environment
     get_credentials: Callable[[], S3StaticCredentials] | None
         Use this function to get and refresh object store credentials
+    force_path_style: bool
+        Whether to force using path-style addressing for buckets
     """
 
     credentials = s3_credentials(
@@ -110,7 +119,12 @@ def s3_storage(
         from_env=from_env,
         get_credentials=get_credentials,
     )
-    options = S3Options(region=region, endpoint_url=endpoint_url, allow_http=allow_http)
+    options = S3Options(
+        region=region,
+        endpoint_url=endpoint_url,
+        allow_http=allow_http,
+        force_path_style=force_path_style,
+    )
     return Storage.new_s3(
         config=options,
         bucket=bucket,
@@ -132,6 +146,7 @@ def s3_object_store_storage(
     expires_after: datetime | None = None,
     anonymous: bool | None = None,
     from_env: bool | None = None,
+    force_path_style: bool = False,
 ) -> Storage:
     credentials = s3_credentials(
         access_key_id=access_key_id,
@@ -142,7 +157,12 @@ def s3_object_store_storage(
         from_env=from_env,
         get_credentials=None,
     )
-    options = S3Options(region=region, endpoint_url=endpoint_url, allow_http=allow_http)
+    options = S3Options(
+        region=region,
+        endpoint_url=endpoint_url,
+        allow_http=allow_http,
+        force_path_style=force_path_style,
+    )
     return Storage.new_s3_object_store(
         config=options,
         bucket=bucket,

icechunk-python/src/config.rs

@@ -351,29 +351,33 @@ pub struct PyS3Options {
     pub allow_http: bool,
     #[pyo3(get, set)]
     pub anonymous: bool,
+    #[pyo3(get, set)]
+    pub force_path_style: bool,
 }
 
 #[pymethods]
 impl PyS3Options {
     #[new]
-    #[pyo3(signature = ( region=None, endpoint_url=None, allow_http=false, anonymous=false))]
+    #[pyo3(signature = ( region=None, endpoint_url=None, allow_http=false, anonymous=false, force_path_style=false))]
     pub(crate) fn new(
         region: Option<String>,
         endpoint_url: Option<String>,
         allow_http: bool,
         anonymous: bool,
+        force_path_style: bool,
     ) -> Self {
-        Self { region, endpoint_url, allow_http, anonymous }
+        Self { region, endpoint_url, allow_http, anonymous, force_path_style }
     }
 
     pub fn __repr__(&self) -> String {
         // TODO: escape
         format!(
-            r#"S3Options(region={region}, endpoint_url={url}, allow_http={http}, anonymous={anon})"#,
+            r#"S3Options(region={region}, endpoint_url={url}, allow_http={http}, anonymous={anon}, force_path_style={force_path_style})"#,
             region = format_option(self.region.as_ref()),
             url = format_option(self.endpoint_url.as_ref()),
             http = format_bool(self.allow_http),
             anon = format_bool(self.anonymous),
+            force_path_style = format_bool(self.force_path_style),
         )
     }
 }
@@ -385,6 +389,7 @@ impl From<&PyS3Options> for S3Options {
             endpoint_url: options.endpoint_url.clone(),
             allow_http: options.allow_http,
             anonymous: options.anonymous,
+            force_path_style: options.force_path_style,
         }
     }
 }
@@ -396,6 +401,7 @@ impl From<S3Options> for PyS3Options {
             endpoint_url: value.endpoint_url,
             allow_http: value.allow_http,
             anonymous: value.anonymous,
+            force_path_style: value.force_path_style,
         }
     }
 }

icechunk-python/tests/run_xarray_backends_tests.py

@@ -85,6 +85,7 @@ def create_zarr_target(self) -> Generator[IcechunkStore]:
             s3_storage(
                 endpoint_url="http://localhost:9000",
                 allow_http=True,
+                force_path_style=True,
                 region="us-east-1",
                 bucket="testbucket",
                 prefix="python-xarray-test__" + str(time.time()),

icechunk-python/tests/test_can_read_old.py

@@ -33,6 +33,7 @@ def mk_repo(create: bool) -> ic.Repository:
         endpoint_url="http://localhost:9000",
         allow_http=True,
         s3_compatible=True,
+        force_path_style=True,
     )
     container = ic.VirtualChunkContainer("s3", "s3://", virtual_store_config)
     config.set_virtual_chunk_container(container)

icechunk-python/tests/test_concurrency.py

@@ -120,6 +120,7 @@ async def test_thread_concurrency() -> None:
         region="us-east-1",
         endpoint_url="http://localhost:9000",
         allow_http=True,
+        force_path_style=True,
         s3_compatible=True,
     )
     container = icechunk.VirtualChunkContainer("s3", "s3://", store_config)
@@ -133,6 +134,7 @@ async def test_thread_concurrency() -> None:
         region="us-east-1",
         endpoint_url="http://localhost:9000",
         allow_http=True,
+        force_path_style=True,
         bucket="testbucket",
         prefix="multithreaded-test__" + str(time.time()),
         access_key_id="minio123",

icechunk-python/tests/test_credentials.py

@@ -26,6 +26,7 @@ def test_refreshable_credentials_grant_access() -> None:
         region="us-east-1",
         endpoint_url="http://localhost:9000",
         allow_http=True,
+        force_path_style=True,
         bucket="testbucket",
         prefix="this-repo-does-not-exist",
         get_credentials=get_good_credentials,
@@ -34,6 +35,7 @@ def test_refreshable_credentials_grant_access() -> None:
         region="us-east-1",
         endpoint_url="http://localhost:9000",
         allow_http=True,
+        force_path_style=True,
         bucket="testbucket",
         prefix="this-repo-does-not-exist",
         get_credentials=get_bad_credentials,
@@ -68,7 +70,10 @@ def test_refreshable_credentials_errors() -> None:
 
     st = Storage.new_s3(
         config=S3Options(
-            region="us-east-1", endpoint_url="http://localhost:9000", allow_http=True
+            region="us-east-1",
+            endpoint_url="http://localhost:9000",
+            allow_http=True,
+            force_path_style=True,
         ),
         bucket="testbucket",
         prefix="this-repo-does-not-exist",
@@ -80,7 +85,10 @@ def test_refreshable_credentials_errors() -> None:
 
     st = Storage.new_s3(
         config=S3Options(
-            region="us-east-1", endpoint_url="http://localhost:9000", allow_http=True
+            region="us-east-1",
+            endpoint_url="http://localhost:9000",
+            allow_http=True,
+            force_path_style=True,
         ),
         bucket="testbucket",
         prefix="this-repo-does-not-exist",
@@ -121,6 +129,7 @@ def test_s3_refreshable_credentials_refresh(tmp_path: Path) -> None:
         region="us-east-1",
         endpoint_url="http://localhost:9000",
         allow_http=True,
+        force_path_style=True,
         bucket="testbucket",
         prefix="this-repo-does-not-exist",
         get_credentials=creds_obj,

icechunk-python/tests/test_distributed_writers.py

@@ -27,6 +27,7 @@ def mk_repo(use_object_store: bool = False) -> icechunk.Repository:
         storage = s3_object_store_storage(
             endpoint_url="http://localhost:9000",
             allow_http=True,
+            force_path_style=True,
             region="us-east-1",
             bucket="testbucket",
             prefix="python-distributed-writers-test__" + str(time.time()),
@@ -37,6 +38,7 @@ def mk_repo(use_object_store: bool = False) -> icechunk.Repository:
         storage = s3_storage(
             endpoint_url="http://localhost:9000",
             allow_http=True,
+            force_path_style=True,
             region="us-east-1",
             bucket="testbucket",
             prefix="python-distributed-writers-test__" + str(time.time()),