Apply ruff/pyupgrade rules (UP) (#1717)

Co-authored-by: Martin Durant <[email protected]>

Author: DimitriPapadopoulos

Diff for: fsspec/asyn.py

@@ -816,11 +816,9 @@ async def _glob(self, path, maxdepth=None, **kwargs):
             p: info
             for p, info in sorted(allpaths.items())
             if pattern.match(
-                (
-                    p + "/"
-                    if append_slash_to_dirname and info["type"] == "directory"
-                    else p
-                )
+                p + "/"
+                if append_slash_to_dirname and info["type"] == "directory"
+                else p
             )
         }
 

Diff for: fsspec/caching.py

@@ -7,9 +7,9 @@
 import os
 import threading
 import warnings
+from concurrent.futures import Future, ThreadPoolExecutor
 from itertools import groupby
 from operator import itemgetter
-from concurrent.futures import Future, ThreadPoolExecutor
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -87,12 +87,7 @@ def _log_stats(self) -> str:
         if self.hit_count == 0 and self.miss_count == 0:
             # a cache that does nothing, this is for logs only
             return ""
-        return " , %s: %d hits, %d misses, %d total requested bytes" % (
-            self.name,
-            self.hit_count,
-            self.miss_count,
-            self.total_requested_bytes,
-        )
+        return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes"
 
     def __repr__(self) -> str:
         # TODO: use rich for better formatting

Diff for: fsspec/implementations/asyn_wrapper.py

@@ -1,6 +1,7 @@
 import asyncio
-import inspect
 import functools
+import inspect
+
 from fsspec.asyn import AsyncFileSystem
 
 

Diff for: fsspec/implementations/reference.py

@@ -176,7 +176,7 @@ def open_refs(field, record):
             try:
                 df = self.pd.read_parquet(data, engine=self.engine)
                 refs = {c: df[c].to_numpy() for c in df.columns}
-            except IOError:
+            except OSError:
                 refs = None
             return refs
 
@@ -431,7 +431,7 @@ def write(self, field, record, base_url=None, storage_options=None):
         if len(partition) < self.record_size:
             try:
                 original = self.open_refs(field, record)
-            except IOError:
+            except OSError:
                 pass
 
         if original:

Diff for: fsspec/implementations/tests/test_asyn_wrapper.py

@@ -1,10 +1,12 @@
 import asyncio
-import pytest
 import os
 
+import pytest
+
 import fsspec
 from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
 from fsspec.implementations.local import LocalFileSystem
+
 from .test_local import csv_files, filetexts
 
 

Diff for: fsspec/implementations/tests/test_smb.py

@@ -29,7 +29,7 @@
 
 
 def stop_docker(container):
-    cmd = shlex.split('docker ps -a -q --filter "name=%s"' % container)
+    cmd = shlex.split(f'docker ps -a -q --filter "name={container}"')
     cid = subprocess.check_output(cmd).strip().decode()
     if cid:
         subprocess.call(["docker", "rm", "-f", "-v", cid])

Diff for: fsspec/implementations/tests/test_zip.py

@@ -105,7 +105,7 @@ def test_zip_glob_star(m):
     outfiles = fs.glob("*")
     assert len(outfiles) == 1
 
-    fn = f"{os.path.dirname(os.path.abspath((__file__)))}/out.zip"
+    fn = f"{os.path.dirname(os.path.abspath(__file__))}/out.zip"
     fs = fsspec.filesystem("zip", fo=fn, mode="r")
     outfiles = fs.glob("*")
     assert len(outfiles) == 1

Diff for: fsspec/spec.py

@@ -10,7 +10,7 @@
 from errno import ESPIPE
 from glob import has_magic
 from hashlib import sha256
-from typing import Any, ClassVar, Dict, Tuple
+from typing import Any, ClassVar
 
 from .callbacks import DEFAULT_CALLBACK
 from .config import apply_config, conf
@@ -117,8 +117,8 @@ class AbstractFileSystem(metaclass=_Cached):
     _extra_tokenize_attributes = ()
 
     # Set by _Cached metaclass
-    storage_args: Tuple[Any, ...]
-    storage_options: Dict[str, Any]
+    storage_args: tuple[Any, ...]
+    storage_options: dict[str, Any]
 
     def __init__(self, *args, **storage_options):
         """Create and configure file-system instance
@@ -615,11 +615,9 @@ def glob(self, path, maxdepth=None, **kwargs):
             p: info
             for p, info in sorted(allpaths.items())
             if pattern.match(
-                (
-                    p + "/"
-                    if append_slash_to_dirname and info["type"] == "directory"
-                    else p
-                )
+                p + "/"
+                if append_slash_to_dirname and info["type"] == "directory"
+                else p
             )
         }
 
@@ -1442,7 +1440,7 @@ def from_json(blob: str) -> AbstractFileSystem:
 
         return json.loads(blob, cls=FilesystemJSONDecoder)
 
-    def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]:
+    def to_dict(self, *, include_password: bool = True) -> dict[str, Any]:
         """
         JSON-serializable dictionary representation of this filesystem instance.
 
@@ -1483,7 +1481,7 @@ def to_dict(self, *, include_password: bool = True) -> Dict[str, Any]:
         )
 
     @staticmethod
-    def from_dict(dct: Dict[str, Any]) -> AbstractFileSystem:
+    def from_dict(dct: dict[str, Any]) -> AbstractFileSystem:
         """
         Recreate a filesystem instance from dictionary representation.
 

Diff for: fsspec/tests/abstract/__init__.py

@@ -225,7 +225,7 @@ def _10_files_with_hashed_names(self, some_fs, some_join, some_path):
         for i in range(10):
             hashed_i = md5(str(i).encode("utf-8")).hexdigest()
             path = some_join(source, f"{hashed_i}.txt")
-            some_fs.pipe(path=path, value=f"{i}".encode("utf-8"))
+            some_fs.pipe(path=path, value=f"{i}".encode())
         return source
 
 

Diff for: fsspec/tests/test_async.py

@@ -208,7 +208,7 @@ async def get_data(self):
 async def test_async_streamed_file_write():
     test_fs = DummyAsyncFS()
     streamed_file = await test_fs.open_async("misc/foo.txt", mode="wb")
-    inp_data = "foo-bar".encode("utf8") * streamed_file.blocksize * 2
+    inp_data = b"foo-bar" * streamed_file.blocksize * 2
     await streamed_file.write(inp_data)
     assert streamed_file.loc == len(inp_data)
     await streamed_file.close()

Diff for: fsspec/tests/test_fuse.py

@@ -121,8 +121,7 @@ def test_chmod(mount_local):
 
     cp = subprocess.run(
         ["cp", str(mount_dir / "text"), str(mount_dir / "new")],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
+        capture_output=True,
         check=False,
     )
 

Diff for: fsspec/tests/test_utils.py

@@ -45,9 +45,7 @@ def test_read_block():
 
 def test_read_block_split_before():
     """Test start/middle/end cases of split_before."""
-    d = (
-        "#header" + "".join(">foo{i}\nFOOBAR{i}\n".format(i=i) for i in range(100000))
-    ).encode()
+    d = ("#header" + "".join(f">foo{i}\nFOOBAR{i}\n" for i in range(100000))).encode()
 
     # Read single record at beginning.
     # All reads include beginning of file and read through termination of

Diff for: fsspec/utils.py

@@ -82,7 +82,8 @@ def infer_storage_options(
         # https://msdn.microsoft.com/en-us/library/jj710207.aspx
         windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path)
         if windows_path:
-            path = "%s:%s" % windows_path.groups()
+            drive, path = windows_path.groups()
+            path = f"{drive}:{path}"
 
     if protocol in ["http", "https"]:
         # for HTTP, we don't want to parse, as requests will anyway

Diff for: pyproject.toml

@@ -184,6 +184,8 @@ select = [
     "RUF024",
     "SIM",
     "SLOT",
+    "SIM101",
+    "UP",
 ]
 ignore = [
 	# Loop control variable `loop` not used within loop body
@@ -208,6 +210,12 @@ ignore = [
     # Fix these codes later
     "G004",
     "PERF203",
+    "UP007",
+    "UP011",
+    "UP015",
+    "UP018",
+    # deprecated
+    "UP027",
     "SIM102",
     "SIM105",
     "SIM108",