Merge pull request #1431 from transformerlab/fix/copy-dir

Fix copy_dir in storage.py to do proper remote -> remote copies

Author: deep1401

api/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
   "soundfile==0.13.1",
   "tensorboardX==2.6.2.2",
   "timm==1.0.15",
-  "transformerlab==0.0.84",
+  "transformerlab==0.0.85",
   "transformerlab-inference==0.2.52",
   "transformers==4.57.1",
   "wandb==0.23.1",

lab-sdk/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab"
-version = "0.0.84"
+version = "0.0.85"
 description = "Python SDK for Transformer Lab"
 readme = "README.md"
 requires-python = ">=3.10"

lab-sdk/src/lab/storage.py

@@ -520,6 +520,12 @@ async def copy_dir(src_dir: str, dest_dir: str) -> None:
     await makedirs(dest_dir, exist_ok=True)
     # Determine the source filesystem independently of destination
     src_fs, _ = _get_fs_for_path(src_dir)
+    # Remember protocol for remote paths so that we can reconstruct full URIs
+    # from keys returned by fsspec (which may omit the protocol).
+    src_protocol: Optional[str] = None
+    if is_remote_path(src_dir):
+        src_protocol = src_dir.split("://", 1)[0]
+
     try:
         src_files = src_fs.find(src_dir)
     except Exception:
@@ -529,8 +535,14 @@ async def copy_dir(src_dir: str, dest_dir: str) -> None:
             for f in files:
                 src_files.append(f)
 
-    for src_file in src_files:
-        # Compute relative path with respect to the source dir
+    for raw_src_file in src_files:
+        # For remote filesystems, ensure we have a full URI (e.g., s3://bucket/...)
+        src_file = raw_src_file
+        if src_protocol is not None and not is_remote_path(raw_src_file):
+            src_file = f"{src_protocol}://{raw_src_file.lstrip('/')}"
+
+        # Compute relative path with respect to the source dir using the
+        # normalized src_file URI/path.
         rel_path = src_file[len(src_dir) :].lstrip("/")
         dest_file = join(dest_dir, rel_path)
         # Ensure destination directory exists