From 2c37c7d27a73ebc118775474c4a0b5cb7c9326d6 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 28 Mar 2023 10:42:57 -0400 Subject: [PATCH 1/3] Set cache_regions to True by default --- s3fs/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/s3fs/core.py b/s3fs/core.py index 5f0faa2b..a54ad4ad 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -223,7 +223,7 @@ class S3FileSystem(AsyncFileSystem): for performance reasons. When set to True, filesystem instances will use the S3 ListObjectVersions API call to list directory contents, which requires listing all historical object versions. - cache_regions : bool (False) + cache_regions : bool (True) Whether to cache bucket regions or not. Whenever a new bucket is used, it will first find out which region it belongs and then use the client for that region. @@ -278,7 +278,7 @@ def __init__( session=None, username=None, password=None, - cache_regions=False, + cache_regions=True, asynchronous=False, loop=None, **kwargs, @@ -899,6 +899,12 @@ async def _makedirs(self, path, exist_ok=False): makedirs = sync_wrapper(_makedirs) async def _rmdir(self, path): + bucket, key = self.split_path(path) + if key: + if await self._exists(path): + # did you mean rm(path, recursive=True) ? + raise FileExistsError + raise FileNotFoundError try: await self._call_s3("delete_bucket", Bucket=path) except botocore.exceptions.ClientError as e: From 67b6f23eef18d75e7f14ba9bed695e8b925221f6 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 28 Mar 2023 10:51:19 -0400 Subject: [PATCH 2/3] fix rmdir --- s3fs/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3fs/core.py b/s3fs/core.py index a54ad4ad..b5caa15a 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -899,8 +899,7 @@ async def _makedirs(self, path, exist_ok=False): makedirs = sync_wrapper(_makedirs) async def _rmdir(self, path): - bucket, key = self.split_path(path) - if key: + if "/" in path: if await self._exists(path): # did you mean rm(path, recursive=True) ? raise FileExistsError From e8e09f5cb6694077333d85d73210f720f97d8797 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 28 Mar 2023 10:52:43 -0400 Subject: [PATCH 3/3] even for full URL --- s3fs/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/s3fs/core.py b/s3fs/core.py index b5caa15a..873e7cb7 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -899,6 +899,7 @@ async def _makedirs(self, path, exist_ok=False): makedirs = sync_wrapper(_makedirs) async def _rmdir(self, path): + path = self._strip_protocol(path).rstrip("/") if "/" in path: if await self._exists(path): # did you mean rm(path, recursive=True) ?