Merge pull request #2021 from transformerlab/fix/storage-makedirs-per-path-fs

Fix path resolution at storage level

Author: deep1401

api/localprovider_pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
   "soundfile==0.13.1",
   "tensorboardX==2.6.2.2",
   "timm==1.0.15",
-  "transformerlab==0.1.32",
+  "transformerlab==0.1.35",
   "transformerlab-inference==0.2.52",
   "transformers==4.57.1",
   "wandb==0.23.1",

api/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
   "python-dotenv==1.1.0",
   "python-multipart==0.0.20",
   "sqlalchemy[asyncio]==2.0.40",
-  "transformerlab==0.1.34",
+  "transformerlab==0.1.35",
   "transformerlab-inference==0.2.52",
   "uvicorn==0.35.0",
   "watchfiles==1.0.5",

lab-sdk/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab"
-version = "0.1.34"
+version = "0.1.35"
 description = "Python SDK for Transformer Lab"
 readme = "README.md"
 requires-python = ">=3.10"

lab-sdk/src/lab/storage.py

@@ -357,7 +357,7 @@ async def root_join(*parts: str) -> str:
 
 
 async def exists(path: str) -> bool:
-    fs = await filesystem()
+    fs, _ = _get_fs_for_path(path)
     try:
         return await asyncio.to_thread(fs.exists, path)
     finally:
@@ -368,7 +368,8 @@ async def exists(path: str) -> bool:
 async def isdir(path: str, fs=None) -> bool:
     filesys = fs
     try:
-        filesys = fs if fs is not None else await filesystem()
+        if fs is None:
+            filesys, _ = _get_fs_for_path(path)
         return await asyncio.to_thread(filesys.isdir, path)
     except Exception:
         return False
@@ -381,7 +382,7 @@ async def isdir(path: str, fs=None) -> bool:
 async def isfile(path: str) -> bool:
     fs = None
     try:
-        fs = await filesystem()
+        fs, _ = _get_fs_for_path(path)
         return await asyncio.to_thread(fs.isfile, path)
     except Exception:
         return False
@@ -391,7 +392,7 @@ async def isfile(path: str) -> bool:
 
 
 async def makedirs(path: str, exist_ok: bool = True) -> None:
-    fs = await filesystem()
+    fs, _ = _get_fs_for_path(path)
     try:
         await asyncio.to_thread(fs.makedirs, path, exist_ok=exist_ok)
     except TypeError:
@@ -404,8 +405,12 @@ async def makedirs(path: str, exist_ok: bool = True) -> None:
 
 
 async def ls(path: str, detail: bool = False, fs=None):
-    # Use provided filesystem or get default
-    filesys = fs if fs is not None else await filesystem()
+    # Use provided filesystem or pick one based on the path's scheme so that
+    # local paths don't get routed through a configured remote filesystem.
+    if fs is not None:
+        filesys = fs
+    else:
+        filesys, _ = _get_fs_for_path(path)
     try:
         paths = await asyncio.to_thread(filesys.ls, path, detail=detail)
         is_remote = path.startswith(_REMOTE_PATH_PREFIXES)