Add ability to mark jobs as discarded and also add scoring

api/localprovider_pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
   "soundfile==0.13.1",
   "tensorboardX==2.6.2.2",
   "timm==1.0.15",
-  "transformerlab==0.1.28",
+  "transformerlab==0.1.29",
   "transformerlab-inference==0.2.52",
   "transformers==4.57.1",
   "wandb==0.23.1",

api/pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
   "python-dotenv==1.1.0",
   "python-multipart==0.0.20",
   "sqlalchemy[asyncio]==2.0.40",
-  "transformerlab==0.1.28",
+  "transformerlab==0.1.29",
   "transformerlab-inference==0.2.52",
   "uvicorn==0.35.0",
   "watchfiles==1.0.5",

api/transformerlab/routers/experiment/jobs.py

@@ -131,10 +131,38 @@ async def job_delete(job_id: str, experimentId: str):
 
 @router.put("/{job_id}/job_data")
 async def job_update_job_data(job_id: str, experimentId: str, body: dict = Body(...)):
-    """Update user-facing metadata fields in job_data (favorite, hidden, tags)."""
+    """Update user-facing metadata fields in job_data (favorite, hidden, tags, discard)."""
     updates = body.get("updates", {})
-    ALLOWED_KEYS = {"favorite", "hidden", "tags"}
-    filtered = {k: v for k, v in updates.items() if k in ALLOWED_KEYS}
+    allowed_keys = {"favorite", "hidden", "tags", "discard"}
+    filtered = {k: v for k, v in updates.items() if k in allowed_keys}
+
+    # Keep discard under job_data.score.discard to avoid introducing a new top-level field.
+    if "discard" in filtered:
+        raw_discard_value = filtered.pop("discard")
+        if isinstance(raw_discard_value, bool):
+            discard_value = raw_discard_value
+        elif isinstance(raw_discard_value, int):
+            if raw_discard_value not in (0, 1):
+                raise HTTPException(status_code=422, detail="discard must be a boolean value")
+            discard_value = bool(raw_discard_value)
+        elif isinstance(raw_discard_value, str):
+            normalized_discard_value = raw_discard_value.strip().lower()
+            if normalized_discard_value in {"true", "false"}:
+                discard_value = normalized_discard_value == "true"
+            elif normalized_discard_value:
+                try:
+                    numeric_discard_value = int(normalized_discard_value)
+                except ValueError as exc:
+                    raise HTTPException(status_code=422, detail="discard must be a boolean value") from exc
+                if numeric_discard_value not in (0, 1):
+                    raise HTTPException(status_code=422, detail="discard must be a boolean value")
+                discard_value = bool(numeric_discard_value)
+            else:
+                raise HTTPException(status_code=422, detail="discard must be a boolean value")
+        else:
+            raise HTTPException(status_code=422, detail="discard must be a boolean value")
+        await job_service.job_update_job_data_score_field(job_id, "discard", discard_value, experimentId)
+
     if not filtered:
         return {"message": "No valid keys to update"}
     await job_service.job_update_job_data_insert_key_values(job_id, filtered, experimentId)

api/transformerlab/services/job_service.py

@@ -402,6 +402,27 @@ async def job_update_job_data_insert_key_values(job_id, updates: Dict[str, Any],
         print(f"Error updating job {job_id}: {e}")
 
 
+async def job_update_job_data_score_field(job_id: str, score_key: str, value: Any, experiment_id: str) -> None:
+    """
+    Merge a single key into job_data.score for a job.
+    """
+    try:
+        resolved_id = await _resolve_full_job_id(str(job_id), str(experiment_id))
+        actual_id = resolved_id or str(job_id)
+
+        job = await Job.get(actual_id, experiment_id)
+        current = (await job.get_job_data()) or {}
+        score = current.get("score") if isinstance(current, dict) else {}
+        merged_score = dict(score) if isinstance(score, dict) else {}
+        merged_score[score_key] = value
+        await job.update_job_data_field("score", merged_score)
+
+        await cache.invalidate(f"job:{actual_id}")
+        await cache.delete(_job_cache_key(actual_id))
+    except Exception as e:
+        print(f"Error updating job {job_id} score field {score_key}: {e}")
+
+
 async def job_stop(job_id, experiment_id):
     print("Stopping job: " + str(job_id))
     await job_update_job_data_insert_key_value(job_id, "stop", True, experiment_id)

cli/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab-cli"
-version = "0.0.44"
+version = "0.0.45"
 description = "Transformer Lab CLI"
 requires-python = ">=3.10"
 authors = [{ name = "Transformer Lab", email = "hello@transformerlab.ai" }]

cli/src/transformerlab_cli/commands/job.py

@@ -248,17 +248,51 @@ def _compute_duration(job_data: dict) -> str:
         return ""
 
 
-def _format_score(score: dict) -> str:
-    """Format a score dict into a compact string for the table view."""
-    if not score or not isinstance(score, dict):
-        return ""
-    parts = [f"{k}={v}" for k, v in score.items() if v is not None]
+def _score_items(score) -> list[tuple[str, object]]:
+    """Normalize score payload to key/value pairs and hide internal flags."""
+    if score is None:
+        return []
+    if isinstance(score, dict):
+        return [
+            (str(key), value) for key, value in score.items() if value is not None and str(key).lower() != "discard"
+        ]
+    return [("score", score)]
+
+
+def _format_score(score) -> str:
+    """Format score payload into a compact string for the table view."""
+    parts = [f"{k}={v}" for k, v in _score_items(score)]
     text = ", ".join(parts)
     if len(text) > 30:
         text = text[:27] + "…"
     return text
 
 
+def _is_discarded(job_data: dict) -> bool:
+    """Return whether a job is marked discarded via score.discard."""
+    if not isinstance(job_data, dict):
+        return False
+    score = job_data.get("score", {})
+    if not isinstance(score, dict):
+        return False
+    discard = score.get("discard", False)
+    if isinstance(discard, bool):
+        return discard
+    if isinstance(discard, int):
+        return discard == 1
+    if isinstance(discard, str):
+        normalized = discard.strip().lower()
+        if normalized == "true":
+            return True
+        if normalized == "false":
+            return False
+        try:
+            return int(normalized) == 1
+        except ValueError:
+            return False
+    return False
+
+
 def _render_jobs(jobs) -> Table:
     """Make a new table."""
     # Create a table to display job details
@@ -279,13 +313,16 @@ def _render_jobs(jobs) -> Table:
         # Truncate long descriptions for the table view
         if description and len(description) > 40:
             description = description[:37] + "…"
+        completion_status = str(job_data.get("completion_status", "N/A"))
+        if _is_discarded(job_data):
+            completion_status = f"{completion_status} (discarded)"
         table.add_row(
             str(job.get("id", "N/A")),
             job.get("experiment_id", "N/A"),
             job_data.get("task_name", "N/A"),
             job.get("status", "N/A"),
             f"{job.get('progress', 0)}%",
-            job_data.get("completion_status", "N/A"),
+            completion_status,
             description or "",
             _format_score(job_data.get("score", {})),
             _compute_duration(job_data),
@@ -364,6 +401,7 @@ def _render_job(job) -> None:
         "End Time": job_data.get("end_time", "N/A"),
         "Duration": _compute_duration(job_data) or "N/A",
         "Completion Status": job_data.get("completion_status", "N/A"),
+        "Discarded": "Yes" if _is_discarded(job_data) else "No",
         "Completion Details": job_data.get("completion_details", "N/A"),
         "Error": job_data.get("error_msg", ""),
         "Config": job_data.get("_config", {}),
@@ -376,16 +414,25 @@ def _render_job(job) -> None:
     details["Config"] = f"\n{config_text}"
 
     score = details.pop("Score")
-    score_text = "\n".join([f"  {key}: {value}" for key, value in score.items()])
-    details["Score"] = f"\n{score_text}"
+    score_items = _score_items(score)
+    if score_items:
+        score_text = "\n".join([f"  {key}: {value}" for key, value in score_items])
+        details["Score"] = f"\n{score_text}"
+    else:
+        details["Score"] = "N/A"
 
     # Display details in a panel
     detail_text = "\n".join([f"[bold]{key}:[/bold] {value}" for key, value in details.items()])
     panel = Panel(detail_text, title=f"Job Details (ID: {job.get('id', 'N/A')})", subtitle=status_chip)
     console.print(panel)
 
 
-def list_jobs(experiment_id: str, running_only: bool = False, sort_by: str | None = None):
+def list_jobs(
+    experiment_id: str,
+    running_only: bool = False,
+    score_metric: str | None = None,
+    score_order: str = "desc",
+):
     """List all jobs for a specific experiment."""
     output_format = cli_state.output_format
     jobs = []
@@ -398,21 +445,22 @@ def list_jobs(experiment_id: str, running_only: bool = False, sort_by: str | Non
     if running_only:
         jobs = [j for j in jobs if j.get("status") in ACTIVE_JOB_STATUSES]
 
-    if sort_by:
-        # Sort by a score key (e.g. "eval/loss"). Jobs without the key go to the end.
+    if score_metric:
+        # Sort by score metric (e.g. "eval/loss"). Missing/non-numeric scores go to the end.
         def _sort_key(job):
             score = job.get("job_data", {}).get("score", {})
-            val = score.get(sort_by) if isinstance(score, dict) else None
+            val = score.get(score_metric) if isinstance(score, dict) else None
             if val is None:
                 return (1, 0.0)  # push to end
             try:
                 return (0, float(val))
             except (ValueError, TypeError):
                 return (1, 0.0)
 
-        jobs = sorted(jobs, key=_sort_key)
+        jobs = sorted(jobs, key=_sort_key, reverse=score_order == "desc")
 
     if output_format == "json":
+        jobs = [{**job, "discarded": _is_discarded(job.get("job_data", {}))} for job in jobs]
         print(json.dumps(jobs))
     else:
         table = _render_jobs(jobs)
@@ -439,7 +487,7 @@ def info_job(job_id: str, experiment_id: str):
 
     if output_format == "json":
         files = _fetch_job_files(experiment_id, job_id)
-        print(json.dumps({**job, "files": files}))
+        print(json.dumps({**job, "files": files, "discarded": _is_discarded(job.get("job_data", {}))}))
         return
 
     console.print(f"[bold success]Job Details for ID {job_id}:[/bold success]")
@@ -863,18 +911,76 @@ def command_job_logs(
     command_job_machine_logs(job_id, follow)
 
 
+@app.command("discard")
+def command_job_discard(
+    job_id: str = typer.Argument(..., help="Job ID to mark as discarded"),
+    undo: bool = typer.Option(False, "--undo", help="Unset discard and mark the job as not discarded"),
+):
+    """Toggle score.discard on a job."""
+    current_experiment = require_current_experiment()
+    discard_value = not undo
+    response = api.put(
+        f"/experiment/{current_experiment}/jobs/{job_id}/job_data",
+        json={"updates": {"discard": discard_value}},
+    )
+    if response.status_code == 200:
+        if cli_state.output_format == "json":
+            print(json.dumps({"job_id": job_id, "discard": discard_value}))
+            return
+        if discard_value:
+            console.print(f"[success]✓[/success] Job [bold]{job_id}[/bold] marked as discarded.")
+        else:
+            console.print(f"[success]✓[/success] Job [bold]{job_id}[/bold] unmarked as discarded.")
+        return
+
+    try:
+        detail = response.json().get("detail", response.text)
+    except Exception:
+        detail = response.text
+    if cli_state.output_format == "json":
+        print(
+            json.dumps(
+                {"error": "Failed to update discard flag", "status_code": response.status_code, "detail": detail}
+            )
+        )
+    else:
+        console.print(
+            f"[error]Error:[/error] Failed to update discard flag for job {job_id}. Status code: {response.status_code}"
+        )
+        if detail:
+            console.print(f"[error]Detail:[/error] {detail}")
+    raise typer.Exit(1)
+
+
 @app.command("list")
 def command_job_list(
     running: bool = typer.Option(
         False, "--running", help="Show only active jobs (WAITING, LAUNCHING, RUNNING, INTERACTIVE)"
     ),
-    sort_by: str = typer.Option(
-        None, "--sort-by", help="Sort jobs by a score metric key (e.g. 'eval/loss'). Ascending order."
+    score_metric: str = typer.Option(
+        None,
+        "--score-metric",
+        "--sort-by",
+        help="Sort jobs by this score metric key (e.g. 'eval/loss').",
+    ),
+    score_order: str = typer.Option(
+        "desc",
+        "--score-order",
+        help="Score ordering direction: desc (default) or asc.",
     ),
 ):
     """List all jobs."""
+    score_order_normalized = score_order.strip().lower()
+    if score_order_normalized not in {"desc", "asc"}:
+        console.print("[error]Error:[/error] --score-order must be either 'desc' or 'asc'.")
+        raise typer.Exit(1)
     current_experiment = require_current_experiment()
-    list_jobs(current_experiment, running_only=running, sort_by=sort_by)
+    list_jobs(
+        current_experiment,
+        running_only=running,
+        score_metric=score_metric,
+        score_order=score_order_normalized,
+    )
 
 
 @app.command("info")