fix comments

Author: deep1401

api/test/api/test_launch_template_juicefs.py

@@ -68,6 +68,7 @@ def test_build_juicefs_pod_config_mount_command_includes_console_url(monkeypatch
 
     assert env_vars["TFL_JUICEFS_CONSOLE_URL"] == "http://juicefs-console:8080"
     assert '--console-url "$TFL_JUICEFS_CONSOLE_URL"' in mount_cmd
+    assert mount_cmd.count('--console-url "$TFL_JUICEFS_CONSOLE_URL"') == 2
 
 
 def test_build_juicefs_pod_config_custom_mount_point(monkeypatch):

api/test/api/test_remote_workspace_juicefs.py

@@ -46,6 +46,18 @@ def test_validate_juicefs_config_raises_when_not_mounted(monkeypatch, tmp_path):
             remote_workspace._validate_juicefs_config()
 
 
+def test_validate_juicefs_config_raises_on_invalid_storage_backend(monkeypatch, tmp_path):
+    monkeypatch.setenv("TFL_JUICEFS_METADATA_URL", "redis://localhost:6379/1")
+    monkeypatch.setenv("TFL_JUICEFS_VOLUME_NAME", "myvol")
+    monkeypatch.setenv("TFL_JUICEFS_STORAGE_BACKEND", "s3")
+    monkeypatch.setenv("TFL_JUICEFS_MOUNT_POINT", str(tmp_path))
+    monkeypatch.setattr(remote_workspace, "STORAGE_PROVIDER", "juicefs", raising=False)
+
+    with patch("transformerlab.shared.remote_workspace.os.path.ismount", return_value=True):
+        with pytest.raises(SystemExit):
+            remote_workspace._validate_juicefs_config()
+
+
 def test_validate_juicefs_config_passes_when_valid(monkeypatch, tmp_path):
     monkeypatch.setenv("TFL_JUICEFS_METADATA_URL", "redis://localhost:6379/1")
     monkeypatch.setenv("TFL_JUICEFS_VOLUME_NAME", "myvol")

api/transformerlab/services/compute_provider/launch_juicefs.py

@@ -61,19 +61,15 @@ def build_juicefs_pod_config(
         f" --subdir {shlex.quote(f'orgs/{team_id}')} --background"
     )
     if juicefs_token:
+        console_flag = ' --console-url "$TFL_JUICEFS_CONSOLE_URL"' if juicefs_console_url else ""
         auth_cmd = (
             'if [ -n "$ACCESS_KEY" ] && [ -n "$SECRET_KEY" ]; then '
-            f'juicefs auth {shlex.quote(volume_name)} --token "$TFL_JUICEFS_TOKEN" '
+            f'juicefs auth {shlex.quote(volume_name)} --token "$TFL_JUICEFS_TOKEN"{console_flag} '
             '--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY"; '
             "else "
-            f'juicefs auth {shlex.quote(volume_name)} --token "$TFL_JUICEFS_TOKEN"; '
+            f'juicefs auth {shlex.quote(volume_name)} --token "$TFL_JUICEFS_TOKEN"{console_flag}; '
             "fi"
         )
-        if juicefs_console_url:
-            auth_cmd = auth_cmd.replace(
-                '--token "$TFL_JUICEFS_TOKEN"',
-                '--token "$TFL_JUICEFS_TOKEN" --console-url "$TFL_JUICEFS_CONSOLE_URL"',
-            )
         mount_cmd = f"{auth_cmd} && {mount_cmd}"
 
     return env_vars, mount_cmd, mount_point

api/transformerlab/shared/remote_workspace.py

@@ -187,6 +187,13 @@ def _validate_juicefs_config() -> None:
         if not os.getenv(required):
             print(f"❌ ERROR: {required} is required when TFL_STORAGE_PROVIDER=juicefs", file=sys.stderr)
             sys.exit(1)
+    backend = os.getenv("TFL_JUICEFS_STORAGE_BACKEND", "")
+    if backend not in ("aws", "gcp", "azure"):
+        print(
+            f"❌ ERROR: Invalid TFL_JUICEFS_STORAGE_BACKEND. Expected one of: aws, gcp, azure; got {backend!r}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
     mount_point = os.getenv("TFL_JUICEFS_MOUNT_POINT", "/mnt/juicefs")
     if not os.path.ismount(mount_point):
         print(

lab-sdk/src/lab/experiment.py

@@ -1,5 +1,7 @@
 import asyncio
 import contextlib
+import os
+import random
 import time
 import uuid
 from datetime import datetime, timezone
@@ -224,12 +226,19 @@ async def create_job(self, type: str = "REMOTE"):
         # filesystems can surface stale metadata/races. Retry with a fresh UUID
         # if a just-created index file appears to already exist.
         new_job = None
-        for _ in range(5):
+        storage_provider = os.getenv("TFL_STORAGE_PROVIDER", "").lower()
+        for attempt in range(5):
             new_job_id = str(uuid.uuid4())
             try:
                 new_job = await Job.create(new_job_id, self.id)
                 break
             except FileExistsError:
+                logger.warning("Job.create raised FileExistsError; retrying with new UUID", exc_info=True)
+                # JuiceFS can briefly surface stale metadata, causing false-positive
+                # "already exists" signals right after creation attempts.
+                if storage_provider == "juicefs" and attempt < 4:
+                    delay_seconds = (0.05 * (2**attempt)) + random.uniform(0.0, 0.05)
+                    await asyncio.sleep(delay_seconds)
                 continue
         if new_job is None:
             raise RuntimeError(f"Unable to create a unique job ID for experiment '{self.id}' after retries")

lab-sdk/tests/test_experiment.py

@@ -199,7 +199,7 @@ async def test_create_job_uses_uuid_and_experiment_dir(tmp_path, monkeypatch):
 
 
 @pytest.mark.asyncio
-async def test_create_job_retries_after_file_exists_error(tmp_path, monkeypatch):
+async def test_create_job_retries_after_file_exists_error(tmp_path, monkeypatch, caplog):
     for mod in list(importlib.sys.modules.keys()):
         if mod.startswith("lab."):
             importlib.sys.modules.pop(mod)
@@ -225,10 +225,53 @@ async def flaky_create(job_id, experiment_id):
 
     monkeypatch.setattr(Job, "create", flaky_create)
 
+    caplog.set_level("WARNING", logger="lab.experiment")
     exp = Experiment("alpha")
     job = await exp.create_job("TRAIN")
     assert str(job.id) == "unique-id"
     assert attempts["count"] == 2
+    assert "Job.create raised FileExistsError; retrying with new UUID" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_create_job_retries_with_backoff_for_juicefs(tmp_path, monkeypatch):
+    for mod in list(importlib.sys.modules.keys()):
+        if mod.startswith("lab."):
+            importlib.sys.modules.pop(mod)
+
+    ws = tmp_path / "ws"
+    ws.mkdir()
+    monkeypatch.setenv("TFL_WORKSPACE_DIR", str(ws))
+    monkeypatch.setenv("TFL_STORAGE_PROVIDER", "juicefs")
+
+    from lab.experiment import Experiment
+    from lab.job import Job
+
+    ids = iter(["collision-id", "unique-id"])
+    monkeypatch.setattr("lab.experiment.uuid.uuid4", lambda: next(ids))
+
+    original_create = Job.create
+    attempts = {"count": 0}
+    sleep_calls = {"count": 0}
+
+    async def flaky_create(job_id, experiment_id):
+        attempts["count"] += 1
+        if attempts["count"] == 1:
+            raise FileExistsError("simulated duplicate job id")
+        return await original_create(job_id, experiment_id)
+
+    async def fake_sleep(_: float):
+        sleep_calls["count"] += 1
+
+    monkeypatch.setattr(Job, "create", flaky_create)
+    monkeypatch.setattr("lab.experiment.asyncio.sleep", fake_sleep)
+    monkeypatch.setattr("lab.experiment.random.uniform", lambda _a, _b: 0.0)
+
+    exp = Experiment("alpha")
+    job = await exp.create_job("TRAIN")
+    assert str(job.id) == "unique-id"
+    assert attempts["count"] == 2
+    assert sleep_calls["count"] == 1
 
 
 @pytest.mark.asyncio