Merge branch 'main' into fix/cli-api-bugs

Author: deep1401

.agents/skills/transformerlab-cli/SKILL.md

@@ -594,9 +594,10 @@ Provider configs (`api_token`, `api_key`, `ssh_key_path`) contain secrets. If th
 8. **Never use `task interactive`** unless the user specifically requests an interactive session
 9. **`job task-logs --follow`** streams continuously and blocks until the job finishes — use when the user wants real-time monitoring
 10. **Never use the deprecated `lab job logs`** — see the "Job logs: three real commands" section below.
-11. **After queuing a task, ASK the user if they'd like you to watch the logs.** Don't start streaming or polling automatically — jobs can take minutes to hours, and `--follow` blocks. Report the Job ID and ask: "Want me to watch the logs and report back?"
-12. **Never create API keys programmatically** — if auth fails, ask the user to provide an API key from the web UI
-13. **Always pass `--description/-m` when queuing a task. Generate it yourself — never ask the user.** See "Always write a run description" below.
+11. **Before queuing a task, CONFIRM the experiment with the user.** Run `lab config` to read the current default and `lab --format json experiment list` to verify it exists, then ask: "I'm about to queue this under experiment `<name>` (your current default). OK, or pick another?" Show 2–3 alternatives from `experiment list` if the current one looks stale or missing. Skip the confirmation only when the user has already named the experiment in this turn.
+12. **After queuing a task, ASK the user if they'd like you to watch the logs.** Don't start streaming or polling automatically — jobs can take minutes to hours, and `--follow` blocks. Report the Job ID and ask: "Want me to watch the logs and report back?"
+13. **Never create API keys programmatically** — if auth fails, ask the user to provide an API key from the web UI
+14. **Always pass `--description/-m` when queuing a task. Generate it yourself — never ask the user.** See "Always write a run description" below.
 
 ### Always write a run description
 

.gitignore

@@ -231,3 +231,4 @@ docs/superpowers/*
 
 /tmp/
 /temp/
+.claude/scheduled_tasks.lock

api/api.py

@@ -370,6 +370,10 @@ async def healthz():
         "storage": {
             "provider": _effective_storage_provider(),
         },
+        # Where the React app is served. In single-origin prod deploys this is
+        # the same as the API origin; in dev it's typically http://localhost:1212.
+        # Used by the CLI to open the /#/cli-auth page during browser login.
+        "frontend_url": os.getenv("FRONTEND_URL") or None,
     }
 
 

api/test/api/test_experiment_jobs.py

@@ -14,6 +14,101 @@ def test_jobs_list_all_endpoints(client):
     resp = client.get("/experiment/alpha/jobs/list?type=TRAIN&status=QUEUED")
     assert resp.status_code in (200, 404)
 
+    # slim mode is opt-in and should still return a 2xx (or 404) response shape
+    resp = client.get("/experiment/alpha/jobs/list?slim=true")
+    assert resp.status_code in (200, 404)
+
+
+def test_slim_job_drops_heavy_keys_only():
+    """_slim_job should remove the denylisted keys but preserve everything else."""
+    from transformerlab.routers.experiment.jobs import _slim_job, _SLIM_JOB_DATA_DROPPED_KEYS
+
+    job = {
+        "id": "abc",
+        "status": "COMPLETE",
+        "type": "TRAIN",
+        "job_data": {
+            # Heavy keys that the list view does not need
+            "cluster_config": {"x": "y"},
+            "env_vars": {"TOKEN": "secret"},
+            "setup": "long shell script",
+            "run": "another long shell script",
+            "parameters": {"lr": 0.001, "batch": 32},
+            "config": {"big": "blob"},
+            "file_mounts": {"/foo": "/bar"},
+            "accelerators": "A100",
+            "cpus": 8,
+            "memory": "32GB",
+            "disk_space": "200GB",
+            "num_nodes": 2,
+            "command": "bash run.sh",
+            "provider_jobs_seen_once": ["a", "b"],
+            "provider_empty_jobs_polls": 5,
+            # Keys the list view does need — these must survive
+            "description": "hello",
+            "start_time": "2025-01-01T00:00:00Z",
+            "score": {"acc": 0.9},
+            "subtype": "interactive",
+            "favorite": True,
+            "hidden": False,
+            "tags": ["a"],
+            "wandb_run_url": "https://wandb.ai/x/y/runs/z",
+            "error_msg": None,
+            "stop_requested": False,
+            "eval_results": ["/path/to/results.csv"],
+            "cached_tunnel_info": {"is_ready": True, "url": "https://x"},
+            "provider_launch_result": {"request_id": "req-1", "instance_ids": ["i-1"]},
+        },
+    }
+
+    slim = _slim_job(job)
+
+    # Top-level keys are untouched
+    assert slim["id"] == "abc"
+    assert slim["status"] == "COMPLETE"
+    assert slim["type"] == "TRAIN"
+
+    slim_data = slim["job_data"]
+    # Every denylisted key must be gone
+    for key in _SLIM_JOB_DATA_DROPPED_KEYS:
+        assert key not in slim_data, f"slim mode leaked heavy key: {key}"
+
+    # Frontend-relevant keys must be preserved verbatim
+    for key in (
+        "description",
+        "start_time",
+        "score",
+        "subtype",
+        "favorite",
+        "hidden",
+        "tags",
+        "wandb_run_url",
+        "error_msg",
+        "stop_requested",
+        "eval_results",
+        "cached_tunnel_info",
+        "provider_launch_result",
+    ):
+        assert key in slim_data, f"slim mode dropped expected key: {key}"
+        assert slim_data[key] == job["job_data"][key]
+
+    # The original job dict is not mutated (defensive copy)
+    assert "cluster_config" in job["job_data"]
+    assert "env_vars" in job["job_data"]
+
+
+def test_slim_job_handles_non_dict_job_data():
+    """_slim_job should be a no-op when job_data is missing or not a dict."""
+    from transformerlab.routers.experiment.jobs import _slim_job
+
+    # Missing job_data
+    job = {"id": "1", "status": "RUNNING"}
+    assert _slim_job(job) == job
+
+    # job_data is a string (legacy/mis-serialized) — leave it alone rather than crash
+    job_str = {"id": "2", "job_data": '{"cluster_config": "x"}'}
+    assert _slim_job(job_str) == job_str
+
 
 def test_job_creation_and_management(client):
     """Test job creation and management endpoints"""

api/transformerlab/routers/experiment/jobs.py

@@ -97,15 +97,55 @@ def _job_logs_not_ready_payload(
     return payload
 
 
+# Heavy job_data keys that polling list-views never need. When ?slim=true is set
+# on /jobs/list, these are stripped from each job's job_data to keep the response
+# small. Per-job detail endpoints still return the full payload. New heavy keys
+# should be added here rather than allowlisting fields, so that adding metadata
+# elsewhere doesn't silently shrink list responses.
+_SLIM_JOB_DATA_DROPPED_KEYS: frozenset[str] = frozenset(
+    {
+        "accelerators",
+        "cluster_config",
+        "command",
+        "config",
+        "cpus",
+        "disk_space",
+        "env_vars",
+        "file_mounts",
+        "memory",
+        "num_nodes",
+        "parameters",
+        "provider_empty_jobs_polls",
+        "provider_jobs_seen_once",
+        "run",
+        "setup",
+    }
+)
+
+
+def _slim_job(job: dict) -> dict:
+    """Return a shallow copy of ``job`` with heavy keys removed from job_data."""
+    job_data = job.get("job_data")
+    if not isinstance(job_data, dict):
+        return job
+    trimmed = {k: v for k, v in job_data.items() if k not in _SLIM_JOB_DATA_DROPPED_KEYS}
+    return {**job, "job_data": trimmed}
+
+
 @router.get("/list")
 async def jobs_get_all(
     experimentId: str,
     type: str = "",
     status: str = "",
     subtype: str = "",
+    slim: bool = False,
 ):
     """
     Return the list of jobs for an experiment, optionally filtered by type/status/subtype.
+
+    When ``slim=true``, heavy provisioning/config fields are stripped from each
+    job's ``job_data`` to keep polling payloads small. The set of stripped keys
+    is defined by ``_SLIM_JOB_DATA_DROPPED_KEYS``.
     """
     jobs = await job_service.jobs_get_all(type=type, status=status, experiment_id=experimentId)
 
@@ -121,7 +161,10 @@ async def jobs_get_all(
                     job_data = {}
             if job_data.get("subtype") == subtype:
                 filtered.append(job)
-        return filtered
+        jobs = filtered
+
+    if slim:
+        jobs = [_slim_job(job) for job in jobs]
 
     return jobs
 

cli/src/transformerlab_cli/commands/login.py

@@ -1,29 +1,43 @@
+import os
+
 import typer
 
 from transformerlab_cli.util.auth import set_api_key, fetch_user_info, fetch_user_teams
+from transformerlab_cli.util.browser_login import (
+    BrowserLoginError,
+    _resolve_frontend_url,
+    run_browser_login,
+)
 from transformerlab_cli.util.config import load_config, set_config
 from transformerlab_cli.util.shared import DEFAULT_BASE_URL, set_base_url
 from transformerlab_cli.util.ui import console
 
 
+def _is_ssh_session() -> bool:
+    return bool(os.environ.get("SSH_CONNECTION") or os.environ.get("SSH_CLIENT") or os.environ.get("SSH_TTY"))
+
+
 app = typer.Typer()
 
 
 @app.command()
 def login(
-    api_key: str = typer.Option(None, "--api-key", help="Your API key"),
+    api_key: str = typer.Option(None, "--api-key", help="API key (skips browser flow; for CI/headless)."),
     server: str = typer.Option(None, "--server", help="Server URL"),
+    no_browser: bool = typer.Option(False, "--no-browser", help="Print the login URL instead of opening a browser."),
+    paste: bool = typer.Option(
+        False,
+        "--paste",
+        help="Skip the loopback server; create an API key in the web UI and paste it here. Auto-enabled over SSH.",
+    ),
 ):
     """Log in to Transformer Lab."""
-    # Load config to set the base URL before attempting login
     config = load_config()
 
-    # Prompt for server URL, showing current value as default so user can confirm or change it
     if not server:
         current_server = config.get("server") or DEFAULT_BASE_URL
         server = typer.prompt("Server URL", default=current_server)
 
-    # Validate and set server URL
     from transformerlab_cli.util.config import _validate_url
 
     normalized_url = _validate_url(server)
@@ -35,60 +49,89 @@ def login(
     server = normalized_url
     set_base_url(server)
 
-    # Save server to config if it changed
     if config.get("server") != server:
         set_config("server", server)
 
-    # Ask for API key if not provided
+    console.print(f"\n[label]Current server:[/label] [value]{server}[/value]")
+
+    browser_team_id = None
+    browser_team_name = None
+
     if not api_key:
-        console.print(
-            f"\n[yellow]You can create an API key at:[/yellow] [bold]{server.rstrip('/')}/#/user/api-keys[/bold]"
-        )
-        api_key = typer.prompt("Please enter your API key", hide_input=True)
+        use_paste = paste or _is_ssh_session()
+        if use_paste:
+            if paste:
+                console.print("[label]Paste mode:[/label] open the API keys page on any device.")
+            else:
+                console.print(
+                    "[label]SSH session detected.[/label] The loopback flow won't work over SSH; "
+                    "switching to paste mode."
+                )
+            frontend_url = _resolve_frontend_url(server)
+            api_keys_url = f"{frontend_url}/#/user/api-keys"
+            console.print(f"\n[label]1.[/label] Open this URL in a browser: [bold]{api_keys_url}[/bold]")
+            console.print("[label]2.[/label] Create a new API key (scoped to the team you want).")
+            console.print("[label]3.[/label] Paste the key below.\n")
+            api_key = typer.prompt("API key", hide_input=True).strip()
+            if not api_key:
+                console.print("[error]Error:[/error] No API key provided.")
+                raise typer.Exit(1)
+        else:
+            # Default path: browser-based login via loopback server.
+            try:
+                result = run_browser_login(server_url=server, open_browser=not no_browser)
+            except BrowserLoginError as e:
+                console.print(f"[error]Error:[/error] {e}")
+                console.print(
+                    "[warning]Tip:[/warning] use [bold]lab login --paste[/bold] (or [bold]--api-key <KEY>[/bold]) "
+                    "for SSH/headless/CI."
+                )
+                raise typer.Exit(1)
 
-    # Attempt login
-    login_success = set_api_key(api_key)
+            api_key = result["api_key"]
+            browser_team_id = result.get("team_id")
+            browser_team_name = result.get("team_name")
 
-    if not login_success:
-        # Even if login fails, show how to change server
+    # Validate the key (works for both paths).
+    if not set_api_key(api_key):
         console.print("\n[warning]To change the server URL, run:[/warning]")
         console.print("[bold]  lab config server <SERVER_URL>[/bold]\n")
-        console.print("[dim]Example: lab config server http://localhost:8000[/dim]")
         raise typer.Exit(1)
 
-    # Fetch user info and teams after successful login
     with console.status("[bold info]Fetching user information...", spinner="dots"):
         user_info = fetch_user_info(api_key)
         teams_info = fetch_user_teams(api_key)
 
     if user_info and teams_info:
-        # Save user email
         user_email = user_info.get("email")
         if user_email:
             set_config("user_email", user_email)
             console.print(f"[success]✓[/success] User email: [label]{user_email}[/label]")
 
-        # Save team info
         teams = teams_info.get("teams", [])
-        if teams:
-            # Prefer the team the API key is scoped to, if provided by the server
-            api_key_team_id = user_info.get("api_key_team_id")
-            selected_team = None
+        selected_team = None
+
+        # 1. Prefer team selected in the browser flow.
+        if browser_team_id:
+            selected_team = next((t for t in teams if t.get("id") == browser_team_id), None)
 
+        # 2. Fall back to api_key_team_id from the server (paste flow).
+        if not selected_team:
+            api_key_team_id = user_info.get("api_key_team_id")
             if api_key_team_id:
                 selected_team = next((t for t in teams if t.get("id") == api_key_team_id), None)
 
-            if not selected_team:
-                # Look for owner role first (case-insensitive)
-                owner_team = next(
-                    (t for t in teams if str(t.get("role", "")).lower() == "owner"),
-                    None,
-                )
-                selected_team = owner_team if owner_team else teams[0]
+        # 3. Fall back to owner team, then first team.
+        if not selected_team and teams:
+            owner_team = next(
+                (t for t in teams if str(t.get("role", "")).lower() == "owner"),
+                None,
+            )
+            selected_team = owner_team if owner_team else teams[0]
 
+        if selected_team:
             team_id = selected_team.get("id")
-            team_name = selected_team.get("name")
-
+            team_name = selected_team.get("name") or browser_team_name
             if team_id:
                 set_config("team_id", team_id)
                 console.print(f"[success]✓[/success] Team ID: [label]{team_id}[/label]")