Merge branch 'add/juicefs-mounting' of https://github.com/transformerlab/transformerlab-app into add/juicefs-mounting

Author: deep1401

api/api.py

@@ -97,6 +97,7 @@ async def lifespan(app: FastAPI):
 
     # Do the following at API Startup:
     print_launch_message()
+    _print_storage_banner()
     # Initialize directories early
     from transformerlab.shared import dirs as shared_dirs
 
@@ -362,6 +363,13 @@ async def healthz():
         "metadata": {
             "email_method": email_method,
         },
+        # Only the provider is exposed here — /healthz is unauthenticated, so
+        # the bucket/path URI must NOT leak. The full URI is logged at startup
+        # (operator-visible) and can be added to authenticated endpoints if
+        # the CLI/UI ever needs it.
+        "storage": {
+            "provider": _effective_storage_provider(),
+        },
     }
 
 
@@ -410,6 +418,46 @@ def print_launch_message():
     print("https://lab.cloud\nhttps://github.com/transformerlab/transformerlab-app\n")
 
 
+def _effective_storage_provider() -> str:
+    """Return the storage backend that's actually in use.
+
+    `TFL_STORAGE_PROVIDER` alone isn't authoritative: when it's set to
+    aws/gcp/azure but `TFL_REMOTE_STORAGE_ENABLED` isn't `true`, the SDK
+    silently falls back to the local filesystem at `~/.transformerlab`.
+    Reporting the configured provider in that case is misleading, so callers
+    should use this helper instead.
+    """
+    from lab.storage import STORAGE_PROVIDER
+
+    if STORAGE_PROVIDER == "localfs":
+        return "localfs"
+    remote_enabled = os.getenv("TFL_REMOTE_STORAGE_ENABLED", "false").lower() == "true"
+    return STORAGE_PROVIDER if remote_enabled else "localfs"
+
+
+def _print_storage_banner():
+    """Print the active storage backend so it's obvious in API logs."""
+    from lab.storage import STORAGE_PROVIDER
+
+    effective = _effective_storage_provider()
+    uri = os.getenv("TFL_STORAGE_URI")
+    if effective == "localfs":
+        # Either configured as localfs, or remote was requested but not enabled —
+        # in both cases the SDK uses ~/.transformerlab/workspace.
+        from lab.dirs import HOME_DIR
+
+        if not uri:
+            uri = os.path.join(HOME_DIR, "orgs", "<org_id>", "workspace")
+    if effective != STORAGE_PROVIDER:
+        # Surface the misconfiguration so operators don't think remote is live.
+        print(
+            f"✅ Storage: {effective} ({uri}) "
+            f"[TFL_STORAGE_PROVIDER={STORAGE_PROVIDER} ignored: TFL_REMOTE_STORAGE_ENABLED is not set]"
+        )
+        return
+    print(f"✅ Storage: {effective} ({uri or 'unset'})")
+
+
 def run():
     args = parse_args()
 

api/test/api/test_server_info.py

@@ -1,3 +1,6 @@
+import pytest
+
+
 def test_is_wsl_false(monkeypatch):
     # Simulate subprocess.CalledProcessError
     import subprocess
@@ -46,3 +49,52 @@ def test_healthz_localfs_mode(client, monkeypatch, tmp_path):
     data = response.json()
     assert data["message"] == "OK"
     assert data["mode"] == "multiuser"
+
+
+def test_healthz_includes_storage_provider_only(client, monkeypatch, tmp_path):
+    """healthz exposes the storage provider but NOT the URI.
+
+    /healthz is unauthenticated, so leaking the bucket/path would be infra
+    reconnaissance fuel for any caller. The full URI is operator-visible
+    in the startup banner only.
+    """
+    monkeypatch.setenv("TFL_STORAGE_URI", str(tmp_path / "storage_root"))
+
+    response = client.get("/healthz")
+    assert response.status_code == 200
+    data = response.json()
+    assert "storage" in data
+    assert "provider" in data["storage"]
+    assert "uri" not in data["storage"], "URI must not be exposed via /healthz (unauthenticated endpoint)"
+
+
+def test_healthz_storage_reports_localfs_when_remote_not_enabled(client, monkeypatch):
+    """When TFL_STORAGE_PROVIDER is cloud but TFL_REMOTE_STORAGE_ENABLED is unset,
+    the SDK falls back to the local filesystem — /healthz must reflect that
+    rather than reporting the configured-but-inactive cloud backend.
+    """
+    # STORAGE_PROVIDER is captured at lab.storage import time, so setenv on
+    # TFL_STORAGE_PROVIDER would be a no-op here — patch the module attribute.
+    import lab.storage
+
+    monkeypatch.setattr(lab.storage, "STORAGE_PROVIDER", "aws")
+    monkeypatch.delenv("TFL_REMOTE_STORAGE_ENABLED", raising=False)
+
+    response = client.get("/healthz")
+    assert response.status_code == 200
+    data = response.json()
+    assert data["storage"]["provider"] == "localfs"
+
+
+@pytest.mark.parametrize("provider", ["aws", "gcp", "azure"])
+def test_healthz_storage_reports_cloud_when_remote_enabled(client, monkeypatch, provider):
+    """When TFL_REMOTE_STORAGE_ENABLED=true, /healthz reports the configured cloud provider."""
+    import lab.storage
+
+    monkeypatch.setattr(lab.storage, "STORAGE_PROVIDER", provider)
+    monkeypatch.setenv("TFL_REMOTE_STORAGE_ENABLED", "true")
+
+    response = client.get("/healthz")
+    assert response.status_code == 200
+    data = response.json()
+    assert data["storage"]["provider"] == provider

cli/src/transformerlab_cli/commands/job.py

@@ -925,9 +925,11 @@ def command_job_discard(
     """Toggle score.discard on a job."""
     current_experiment = require_current_experiment()
     discard_value = not undo
+    payload = json.dumps({"updates": {"discard": discard_value}}).encode("utf-8")
     response = api.put(
         f"/experiment/{current_experiment}/jobs/{job_id}/job_data",
-        json={"updates": {"discard": discard_value}},
+        content=payload,
+        headers={"Content-Type": "application/json"},
     )
     if response.status_code == 200:
         if cli_state.output_format == "json":

cli/src/transformerlab_cli/util/config.py

@@ -20,6 +20,32 @@
 # to avoid repeated file reads
 cached_config = None
 
+# Cached /healthz response so the header doesn't refetch per-process.
+# Sentinel: None = not yet attempted; {} = fetch failed (don't retry).
+_cached_healthz: dict[str, Any] | None = None
+
+
+def _get_storage_label() -> str:
+    """Return a short storage backend label for the header, e.g. 'aws' or '?'."""
+    global _cached_healthz
+    if _cached_healthz is None:
+        _cached_healthz = {}
+        try:
+            # Imported lazily so config import doesn't pull httpx at startup
+            from transformerlab_cli.util.api import get
+
+            response = get("/healthz", timeout=2.0)
+            if response.status_code == 200:
+                _cached_healthz = response.json() or {}
+        except Exception:
+            _cached_healthz = {}
+
+    storage = _cached_healthz.get("storage") if isinstance(_cached_healthz, dict) else None
+    if not isinstance(storage, dict):
+        return "?"
+    provider = storage.get("provider")
+    return str(provider) if provider else "?"
+
 
 def load_config() -> dict[str, Any]:
     """Load config from file, return empty dict if not found.
@@ -222,15 +248,17 @@ def check_configs(output_format: str = "pretty") -> None:
     team_name = config.get("team_id", "N/A")
     server = config.get("server", "N/A")
     experiment = config.get("current_experiment", "N/A")
+    storage = _get_storage_label()
 
     table = Table(show_header=True, header_style="header", box=None, title_justify="left")
 
     table.add_column("User Email", style="value")
     table.add_column("Team ID", style="value")
     table.add_column("Server", style="value")
     table.add_column("Experiment", style="value")
+    table.add_column("Storage", style="value")
 
-    table.add_row(user_email, team_name, server, experiment)
+    table.add_row(user_email, team_name, server, experiment, storage)
     console.rule()
     one_liner_logo(console)
     console.print(table)

cli/tests/commands/test_job.py

@@ -612,10 +612,10 @@ def test_job_discard_sets_discard_true(_mock_require, mock_put):
 
     result = runner.invoke(app, ["job", "discard", "42"])
     assert result.exit_code == 0
-    mock_put.assert_called_once_with(
-        "/experiment/exp1/jobs/42/job_data",
-        json={"updates": {"discard": True}},
-    )
+    mock_put.assert_called_once()
+    assert mock_put.call_args.args[0] == "/experiment/exp1/jobs/42/job_data"
+    assert mock_put.call_args.kwargs["headers"] == {"Content-Type": "application/json"}
+    assert json.loads(mock_put.call_args.kwargs["content"].decode("utf-8")) == {"updates": {"discard": True}}
 
 
 @patch("transformerlab_cli.commands.job.api.put")
@@ -627,10 +627,10 @@ def test_job_discard_undo_sets_discard_false(_mock_require, mock_put):
 
     result = runner.invoke(app, ["job", "discard", "42", "--undo"])
     assert result.exit_code == 0
-    mock_put.assert_called_once_with(
-        "/experiment/exp1/jobs/42/job_data",
-        json={"updates": {"discard": False}},
-    )
+    mock_put.assert_called_once()
+    assert mock_put.call_args.args[0] == "/experiment/exp1/jobs/42/job_data"
+    assert mock_put.call_args.kwargs["headers"] == {"Content-Type": "application/json"}
+    assert json.loads(mock_put.call_args.kwargs["content"].decode("utf-8")) == {"updates": {"discard": False}}
 
 
 # ---------------------------------------------------------------------------