Improve Performance on LocalFileSystem.ls() if detail=False (#1789)

Co-authored-by: Martin Durant <[email protected]>

Author: FrankEssenberger

fsspec/implementations/local.py

@@ -57,20 +57,21 @@ def rmdir(self, path):
 
     def ls(self, path, detail=False, **kwargs):
         path = self._strip_protocol(path)
-        info = self.info(path)
-        if info["type"] == "directory":
+        path_info = self.info(path)
+        infos = []
+        if path_info["type"] == "directory":
             with os.scandir(path) as it:
-                infos = []
                 for f in it:
                     try:
-                        infos.append(self.info(f))
+                        # Only get the info if requested since it is a bit expensive (the stat call inside)
+                        # The strip_protocol is also used in info() and calls make_path_posix to always return posix paths
+                        info = self.info(f) if detail else self._strip_protocol(f.path)
+                        infos.append(info)
                     except FileNotFoundError:
                         pass
         else:
-            infos = [info]
+            infos = [path_info] if detail else [path_info["name"]]
 
-        if not detail:
-            return [i["name"] for i in infos]
         return infos
 
     def info(self, path, **kwargs):

fsspec/implementations/tests/test_local.py

@@ -402,6 +402,21 @@ def test_ls_on_file(tmpdir):
     assert fs.exists(resource)
     assert fs.ls(tmpdir) == fs.ls(resource)
     assert fs.ls(resource, detail=True)[0] == fs.info(resource)
+    assert fs.ls(resource, detail=False)[0] == resource
+
+
+def test_ls_on_files(tmpdir):
+    tmpdir = make_path_posix(str(tmpdir))
+    fs = LocalFileSystem()
+    resources = [f"{tmpdir}/{file}" for file in ["file_1.json", "file_2.json"]]
+    for r in resources:
+        fs.touch(r)
+
+    actual_files = fs.ls(tmpdir, detail=True)
+    assert {f["name"] for f in actual_files} == set(resources)
+
+    actual_files = fs.ls(tmpdir, detail=False)
+    assert set(actual_files) == set(resources)
 
 
 @pytest.mark.parametrize("file_protocol", ["", "file://"])