From fc722a76c3b8b623859853093fff0d108e8e3b0e Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:14:06 -0600 Subject: [PATCH 1/8] Add ability to mark jobs as discarded --- api/localprovider_pyproject.toml | 2 +- api/pyproject.toml | 2 +- api/transformerlab/routers/experiment/jobs.py | 17 +++++- cli/pyproject.toml | 2 +- cli/src/transformerlab_cli/commands/job.py | 57 +++++++++++++++++- cli/tests/commands/test_job.py | 60 ++++++++++++++++++- cli/uv.lock | 2 +- lab-sdk/pyproject.toml | 2 +- lab-sdk/src/lab/lab_facade.py | 7 ++- lab-sdk/tests/test_lab_facade.py | 22 ++++++- .../components/Experiment/Tasks/JobsList.tsx | 21 +++++++ .../components/Experiment/Tasks/Tasks.tsx | 34 +++++++++++ 12 files changed, 214 insertions(+), 14 deletions(-) diff --git a/api/localprovider_pyproject.toml b/api/localprovider_pyproject.toml index 75d3c59127..66521aebc6 100644 --- a/api/localprovider_pyproject.toml +++ b/api/localprovider_pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "soundfile==0.13.1", "tensorboardX==2.6.2.2", "timm==1.0.15", - "transformerlab==0.1.28", + "transformerlab==0.1.29", "transformerlab-inference==0.2.52", "transformers==4.57.1", "wandb==0.23.1", diff --git a/api/pyproject.toml b/api/pyproject.toml index e02157d4c9..c8742847c5 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "python-dotenv==1.1.0", "python-multipart==0.0.20", "sqlalchemy[asyncio]==2.0.40", - "transformerlab==0.1.28", + "transformerlab==0.1.29", "transformerlab-inference==0.2.52", "uvicorn==0.35.0", "watchfiles==1.0.5", diff --git a/api/transformerlab/routers/experiment/jobs.py b/api/transformerlab/routers/experiment/jobs.py index 24dee025b9..e9f479f805 100644 --- a/api/transformerlab/routers/experiment/jobs.py +++ b/api/transformerlab/routers/experiment/jobs.py @@ -131,10 +131,21 @@ async def job_delete(job_id: str, experimentId: str): @router.put("/{job_id}/job_data") async def job_update_job_data(job_id: str, experimentId: str, body: dict = Body(...)): - """Update user-facing metadata fields in job_data (favorite, hidden, tags).""" + """Update user-facing metadata fields in job_data (favorite, hidden, tags, discard).""" updates = body.get("updates", {}) - ALLOWED_KEYS = {"favorite", "hidden", "tags"} - filtered = {k: v for k, v in updates.items() if k in ALLOWED_KEYS} + allowed_keys = {"favorite", "hidden", "tags", "discard"} + filtered = {k: v for k, v in updates.items() if k in allowed_keys} + + # Keep discard under job_data.score.discard to avoid introducing a new top-level field. + if "discard" in filtered: + discard_value = bool(filtered.pop("discard")) + existing_job = await job_service.job_get_cached(job_id, experiment_id=experimentId) + existing_job_data = (existing_job or {}).get("job_data", {}) if isinstance(existing_job, dict) else {} + existing_score = existing_job_data.get("score", {}) if isinstance(existing_job_data, dict) else {} + merged_score = dict(existing_score) if isinstance(existing_score, dict) else {} + merged_score["discard"] = discard_value + filtered["score"] = merged_score + if not filtered: return {"message": "No valid keys to update"} await job_service.job_update_job_data_insert_key_values(job_id, filtered, experimentId) diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 4314d5da9b..f294e283e6 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "transformerlab-cli" -version = "0.0.44" +version = "0.0.45" description = "Transformer Lab CLI" requires-python = ">=3.10" authors = [{ name = "Transformer Lab", email = "hello@transformerlab.ai" }] diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index 78c0e4ebd2..28a37aa979 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -259,6 +259,16 @@ def _format_score(score: dict) -> str: return text +def _is_discarded(job_data: dict) -> bool: + """Return whether a job is marked discarded via score.discard.""" + if not isinstance(job_data, dict): + return False + score = job_data.get("score", {}) + if not isinstance(score, dict): + return False + return bool(score.get("discard", False)) + + def _render_jobs(jobs) -> Table: """Make a new table.""" # Create a table to display job details @@ -279,13 +289,16 @@ def _render_jobs(jobs) -> Table: # Truncate long descriptions for the table view if description and len(description) > 40: description = description[:37] + "…" + completion_status = str(job_data.get("completion_status", "N/A")) + if _is_discarded(job_data): + completion_status = f"{completion_status} (discarded)" table.add_row( str(job.get("id", "N/A")), job.get("experiment_id", "N/A"), job_data.get("task_name", "N/A"), job.get("status", "N/A"), f"{job.get('progress', 0)}%", - job_data.get("completion_status", "N/A"), + completion_status, description or "", _format_score(job_data.get("score", {})), _compute_duration(job_data), @@ -364,6 +377,7 @@ def _render_job(job) -> None: "End Time": job_data.get("end_time", "N/A"), "Duration": _compute_duration(job_data) or "N/A", "Completion Status": job_data.get("completion_status", "N/A"), + "Discarded": "Yes" if _is_discarded(job_data) else "No", "Completion Details": job_data.get("completion_details", "N/A"), "Error": job_data.get("error_msg", ""), "Config": job_data.get("_config", {}), @@ -413,6 +427,7 @@ def _sort_key(job): jobs = sorted(jobs, key=_sort_key) if output_format == "json": + jobs = [{**job, "discarded": _is_discarded(job.get("job_data", {}))} for job in jobs] print(json.dumps(jobs)) else: table = _render_jobs(jobs) @@ -439,7 +454,7 @@ def info_job(job_id: str, experiment_id: str): if output_format == "json": files = _fetch_job_files(experiment_id, job_id) - print(json.dumps({**job, "files": files})) + print(json.dumps({**job, "files": files, "discarded": _is_discarded(job.get("job_data", {}))})) return console.print(f"[bold success]Job Details for ID {job_id}:[/bold success]") @@ -863,6 +878,44 @@ def command_job_logs( command_job_machine_logs(job_id, follow) +@app.command("discard") +def command_job_discard( + job_id: str = typer.Argument(..., help="Job ID to mark as discarded"), + undo: bool = typer.Option(False, "--undo", help="Unset discard and mark the job as not discarded"), +): + """Toggle score.discard on a job.""" + current_experiment = require_current_experiment() + discard_value = not undo + response = api.put( + f"/experiment/{current_experiment}/jobs/{job_id}/job_data", + json={"updates": {"discard": discard_value}}, + ) + if response.status_code == 200: + if cli_state.output_format == "json": + print(json.dumps({"job_id": job_id, "discard": discard_value})) + return + if discard_value: + console.print(f"[success]✓[/success] Job [bold]{job_id}[/bold] marked as discarded.") + else: + console.print(f"[success]✓[/success] Job [bold]{job_id}[/bold] unmarked as discarded.") + return + + try: + detail = response.json().get("detail", response.text) + except Exception: + detail = response.text + if cli_state.output_format == "json": + print(json.dumps({"error": "Failed to update discard flag", "status_code": response.status_code, "detail": detail})) + else: + console.print( + f"[error]Error:[/error] Failed to update discard flag for job {job_id}. " + f"Status code: {response.status_code}" + ) + if detail: + console.print(f"[error]Detail:[/error] {detail}") + raise typer.Exit(1) + + @app.command("list") def command_job_list( running: bool = typer.Option( diff --git a/cli/tests/commands/test_job.py b/cli/tests/commands/test_job.py index 87a8d70fe9..ef0437fb61 100644 --- a/cli/tests/commands/test_job.py +++ b/cli/tests/commands/test_job.py @@ -29,7 +29,7 @@ "task_name": "eval", "completion_status": "SUCCESS", "description": "Eval on test split", - "score": {"eval/loss": 2.1, "accuracy": 0.95}, + "score": {"eval/loss": 2.1, "accuracy": 0.95, "discard": True}, "start_time": "2026-04-24 10:00:00", "end_time": "2026-04-24 10:05:30", }, @@ -211,6 +211,16 @@ def test_job_list_shows_score(_mock_check, _mock_require, _mock_api): assert "Score" in out +@patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +def test_job_info_pretty_shows_discarded(_mock_require, _mock_api): + """job info pretty output should include discarded status.""" + result = runner.invoke(app, ["job", "info", "2"]) + assert result.exit_code == 0 + out = strip_ansi(result.output) + assert "discarded" in out.lower() + + @patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) @patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") @patch("transformerlab_cli.commands.job.check_configs") @@ -257,6 +267,19 @@ def test_job_info_json_output(_mock_require, _mock_api): assert data["id"] == 1 assert data["status"] == "RUNNING" assert data["files"] == [{"name": "out.log", "is_dir": False, "size": 42}] + assert data["discarded"] is False + + +@patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +@patch("transformerlab_cli.commands.job.check_configs") +def test_job_list_json_output_includes_discarded(_mock_check, _mock_require, _mock_api): + """job list --format json should include explicit discarded flag per job.""" + result = runner.invoke(app, ["--format", "json", "job", "list"]) + assert result.exit_code == 0 + data = json.loads(result.output.strip()) + assert isinstance(data, list) + assert all("discarded" in job for job in data) @patch( @@ -545,6 +568,41 @@ def test_deprecated_logs_still_works(_mock_require, _mock_fetch): assert "line1" in result.output +# --------------------------------------------------------------------------- +# Discard command tests +# --------------------------------------------------------------------------- + + +@patch("transformerlab_cli.commands.job.api.put") +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +def test_job_discard_sets_discard_true(_mock_require, mock_put): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_put.return_value = mock_resp + + result = runner.invoke(app, ["job", "discard", "42"]) + assert result.exit_code == 0 + mock_put.assert_called_once_with( + "/experiment/exp1/jobs/42/job_data", + json={"updates": {"discard": True}}, + ) + + +@patch("transformerlab_cli.commands.job.api.put") +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +def test_job_discard_undo_sets_discard_false(_mock_require, mock_put): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_put.return_value = mock_resp + + result = runner.invoke(app, ["job", "discard", "42", "--undo"]) + assert result.exit_code == 0 + mock_put.assert_called_once_with( + "/experiment/exp1/jobs/42/job_data", + json={"updates": {"discard": False}}, + ) + + # --------------------------------------------------------------------------- # Publish command tests # --------------------------------------------------------------------------- diff --git a/cli/uv.lock b/cli/uv.lock index 60d80ad8b5..dfd82d76cc 100644 --- a/cli/uv.lock +++ b/cli/uv.lock @@ -371,7 +371,7 @@ wheels = [ [[package]] name = "transformerlab-cli" -version = "0.0.44" +version = "0.0.45" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/lab-sdk/pyproject.toml b/lab-sdk/pyproject.toml index 1ab2581a55..24a6629889 100644 --- a/lab-sdk/pyproject.toml +++ b/lab-sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "transformerlab" -version = "0.1.28" +version = "0.1.29" description = "Python SDK for Transformer Lab" readme = "README.md" requires-python = ">=3.10" diff --git a/lab-sdk/src/lab/lab_facade.py b/lab-sdk/src/lab/lab_facade.py index e951d99f03..d5cccd3ad6 100644 --- a/lab-sdk/src/lab/lab_facade.py +++ b/lab-sdk/src/lab/lab_facade.py @@ -638,8 +638,11 @@ def finish( except Exception: # Never let optional Trackio integration break finish() logger.debug("Trackio integration failed during finish()", exc_info=True) - if score is not None: - _run_async(self._job.update_job_data_field("score", score)) # type: ignore[union-attr] + resolved_score: Dict[str, Any] = {} + if isinstance(score, dict): + resolved_score.update(score) + resolved_score.setdefault("discard", False) + _run_async(self._job.update_job_data_field("score", resolved_score)) # type: ignore[union-attr] if additional_output_path is not None and additional_output_path.strip() != "": _run_async(self._job.update_job_data_field("additional_output_path", additional_output_path)) # type: ignore[union-attr] if plot_data_path is not None and plot_data_path.strip() != "": diff --git a/lab-sdk/tests/test_lab_facade.py b/lab-sdk/tests/test_lab_facade.py index da3fb11e52..ed67b2dbd6 100644 --- a/lab-sdk/tests/test_lab_facade.py +++ b/lab-sdk/tests/test_lab_facade.py @@ -204,7 +204,27 @@ def test_lab_finish(tmp_path, monkeypatch): job_data = lab.get_job_data() assert job_data["completion_status"] == "success" assert job_data["completion_details"] == "Job completed" - assert job_data["score"] == {"accuracy": 0.95} + assert job_data["score"] == {"accuracy": 0.95, "discard": False} + + +def test_lab_finish_preserves_explicit_discard_value(tmp_path, monkeypatch): + _fresh(monkeypatch) + home = tmp_path / ".tfl_home" + ws = tmp_path / ".tfl_ws" + home.mkdir() + ws.mkdir() + monkeypatch.setenv("TFL_HOME_DIR", str(home)) + monkeypatch.setenv("TFL_WORKSPACE_DIR", str(ws)) + + from lab.lab_facade import Lab + + lab = Lab() + lab.init(experiment_id="test_exp") + + lab.finish(message="Job completed", score={"accuracy": 0.95, "discard": True}) + + job_data = lab.get_job_data() + assert job_data["score"] == {"accuracy": 0.95, "discard": True} def test_lab_finish_with_paths(tmp_path, monkeypatch): diff --git a/src/renderer/components/Experiment/Tasks/JobsList.tsx b/src/renderer/components/Experiment/Tasks/JobsList.tsx index bae8a1c93e..8b0395f3ef 100644 --- a/src/renderer/components/Experiment/Tasks/JobsList.tsx +++ b/src/renderer/components/Experiment/Tasks/JobsList.tsx @@ -24,6 +24,7 @@ import { EyeOffIcon, EyeIcon, LinkIcon, + BanIcon, } from 'lucide-react'; import { Typography } from '@mui/joy'; import { @@ -63,6 +64,7 @@ interface JobsListProps { onToggleJobSelected?: (jobId: string) => void; onToggleFavorite?: (jobId: string, currentValue: boolean) => void; onToggleHidden?: (jobId: string, currentValue: boolean) => void; + onToggleDiscard?: (jobId: string, currentValue: boolean) => void; hideJobId?: boolean; showInteractiveType?: boolean; showFilesButton?: boolean; @@ -92,6 +94,7 @@ const JobsList: React.FC = ({ onToggleJobSelected, onToggleFavorite, onToggleHidden, + onToggleDiscard, hideJobId = false, showInteractiveType = false, showFilesButton = true, @@ -607,6 +610,24 @@ const JobsList: React.FC = ({ )} + + onToggleDiscard?.( + String(job.id), + !!job?.job_data?.score?.discard, + ) + } + > + {job?.job_data?.score?.discard ? ( + <> + Unmark discard + + ) : ( + <> + Mark discard + + )} + )} diff --git a/src/renderer/components/Experiment/Tasks/Tasks.tsx b/src/renderer/components/Experiment/Tasks/Tasks.tsx index 93568354b6..3696f042e4 100644 --- a/src/renderer/components/Experiment/Tasks/Tasks.tsx +++ b/src/renderer/components/Experiment/Tasks/Tasks.tsx @@ -656,6 +656,39 @@ export default function Tasks({ subtype }: { subtype?: string }) { } }; + const handleToggleDiscard = async (jobId: string, currentValue: boolean) => { + if (!experimentInfo?.id) return; + const newValue = !currentValue; + const currentJob = jobsWithPlaceholders.find( + (job: any) => String(job?.id) === String(jobId), + ); + const currentScore = + currentJob?.job_data && typeof currentJob.job_data === 'object' + ? currentJob.job_data.score + : {}; + const nextScore = { + ...(typeof currentScore === 'object' && currentScore ? currentScore : {}), + discard: newValue, + }; + updateJobDataOptimistic(jobId, 'score', nextScore); + try { + await fetchWithAuth( + chatAPI.Endpoints.Jobs.UpdateJobData(experimentInfo.id, jobId), + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ updates: { discard: newValue } }), + }, + ); + } catch (error) { + console.error('Error toggling discard:', error); + updateJobDataOptimistic(jobId, 'score', { + ...(typeof currentScore === 'object' && currentScore ? currentScore : {}), + discard: currentValue, + }); + } + }; + const handleExportToTeamGallery = async (taskId: string) => { try { const response = await fetchWithAuth( @@ -1492,6 +1525,7 @@ export default function Tasks({ subtype }: { subtype?: string }) { }} onToggleFavorite={handleToggleFavorite} onToggleHidden={handleToggleHidden} + onToggleDiscard={handleToggleDiscard} showFilesButton={false} forceArtifactsButtonVisible onStopPendingChange={handleStopPendingChange} From b66dcdd20a1c6ddd817fc809d0a72b14d5114bc4 Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:17:53 -0600 Subject: [PATCH 2/8] format --- cli/src/transformerlab_cli/commands/job.py | 9 ++++++--- src/renderer/components/Experiment/Tasks/Tasks.tsx | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index 28a37aa979..2a38715503 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -905,11 +905,14 @@ def command_job_discard( except Exception: detail = response.text if cli_state.output_format == "json": - print(json.dumps({"error": "Failed to update discard flag", "status_code": response.status_code, "detail": detail})) + print( + json.dumps( + {"error": "Failed to update discard flag", "status_code": response.status_code, "detail": detail} + ) + ) else: console.print( - f"[error]Error:[/error] Failed to update discard flag for job {job_id}. " - f"Status code: {response.status_code}" + f"[error]Error:[/error] Failed to update discard flag for job {job_id}. Status code: {response.status_code}" ) if detail: console.print(f"[error]Detail:[/error] {detail}") diff --git a/src/renderer/components/Experiment/Tasks/Tasks.tsx b/src/renderer/components/Experiment/Tasks/Tasks.tsx index 3696f042e4..0f6b5cbfa7 100644 --- a/src/renderer/components/Experiment/Tasks/Tasks.tsx +++ b/src/renderer/components/Experiment/Tasks/Tasks.tsx @@ -683,7 +683,9 @@ export default function Tasks({ subtype }: { subtype?: string }) { } catch (error) { console.error('Error toggling discard:', error); updateJobDataOptimistic(jobId, 'score', { - ...(typeof currentScore === 'object' && currentScore ? currentScore : {}), + ...(typeof currentScore === 'object' && currentScore + ? currentScore + : {}), discard: currentValue, }); } From 590c74e8bca1d3978489030a0fd0e45ec73a61e4 Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:41:01 -0600 Subject: [PATCH 3/8] Fix scoring in CLI and add scoring on jobs list --- cli/src/transformerlab_cli/commands/job.py | 29 +++++-- cli/tests/commands/test_job.py | 15 ++++ .../components/Experiment/Tasks/JobsList.tsx | 87 +++++++++++++++++++ 3 files changed, 124 insertions(+), 7 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index 2a38715503..30a90aab71 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -248,11 +248,22 @@ def _compute_duration(job_data: dict) -> str: return "" -def _format_score(score: dict) -> str: - """Format a score dict into a compact string for the table view.""" - if not score or not isinstance(score, dict): - return "" - parts = [f"{k}={v}" for k, v in score.items() if v is not None] +def _score_items(score) -> list[tuple[str, object]]: + """Normalize score payload to key/value pairs and hide internal flags.""" + if score is None: + return [] + if isinstance(score, dict): + return [ + (str(key), value) + for key, value in score.items() + if value is not None and str(key).lower() != "discard" + ] + return [("score", score)] + + +def _format_score(score) -> str: + """Format score payload into a compact string for the table view.""" + parts = [f"{k}={v}" for k, v in _score_items(score)] text = ", ".join(parts) if len(text) > 30: text = text[:27] + "…" @@ -390,8 +401,12 @@ def _render_job(job) -> None: details["Config"] = f"\n{config_text}" score = details.pop("Score") - score_text = "\n".join([f" {key}: {value}" for key, value in score.items()]) - details["Score"] = f"\n{score_text}" + score_items = _score_items(score) + if score_items: + score_text = "\n".join([f" {key}: {value}" for key, value in score_items]) + details["Score"] = f"\n{score_text}" + else: + details["Score"] = "N/A" # Display details in a panel detail_text = "\n".join([f"[bold]{key}:[/bold] {value}" for key, value in details.items()]) diff --git a/cli/tests/commands/test_job.py b/cli/tests/commands/test_job.py index ef0437fb61..791d6abf8a 100644 --- a/cli/tests/commands/test_job.py +++ b/cli/tests/commands/test_job.py @@ -209,6 +209,8 @@ def test_job_list_shows_score(_mock_check, _mock_require, _mock_api): assert "eval/" in out # Job 1 (no score) should have empty score column — just verify Score header is present assert "Score" in out + # discard flag should not be shown as a score metric + assert "discard" not in out.lower() @patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) @@ -221,6 +223,19 @@ def test_job_info_pretty_shows_discarded(_mock_require, _mock_api): assert "discarded" in out.lower() +@patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +def test_job_info_pretty_shows_score_without_discard(_mock_require, _mock_api): + """job info pretty output should show score metrics and hide score.discard.""" + result = runner.invoke(app, ["job", "info", "2"]) + assert result.exit_code == 0 + out = strip_ansi(result.output) + assert "Score:" in out + assert "eval/loss" in out + assert "accuracy" in out + assert "discard: True" not in out + + @patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) @patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") @patch("transformerlab_cli.commands.job.check_configs") diff --git a/src/renderer/components/Experiment/Tasks/JobsList.tsx b/src/renderer/components/Experiment/Tasks/JobsList.tsx index 8b0395f3ef..f7f9ad02c7 100644 --- a/src/renderer/components/Experiment/Tasks/JobsList.tsx +++ b/src/renderer/components/Experiment/Tasks/JobsList.tsx @@ -11,6 +11,7 @@ import MenuItem from '@mui/joy/MenuItem'; import Dropdown from '@mui/joy/Dropdown'; import Checkbox from '@mui/joy/Checkbox'; import Tooltip from '@mui/joy/Tooltip'; +import Chip from '@mui/joy/Chip'; import { Trash2Icon, LineChartIcon, @@ -122,6 +123,61 @@ const JobsList: React.FC = ({ return type.charAt(0).toUpperCase() + type.slice(1); }; + const formatScoreValue = (value: number): string => { + if (!Number.isFinite(value)) return String(value); + if (Math.abs(value) >= 1000) return value.toFixed(1); + if (Math.abs(value) >= 100) return value.toFixed(2); + if (Math.abs(value) >= 1) return value.toFixed(3); + return value.toFixed(4); + }; + + const getScoreDisplay = ( + score: unknown, + ): { label: string; tooltip?: string } | null => { + if (typeof score === 'number') { + return { label: `Score: ${formatScoreValue(score)}` }; + } + + if (typeof score === 'string') { + const parsed = Number.parseFloat(score); + if (Number.isFinite(parsed)) { + return { label: `Score: ${formatScoreValue(parsed)}` }; + } + if (score.trim()) { + return { label: `Score: ${score.trim()}` }; + } + return null; + } + + if (score && typeof score === 'object') { + const numericEntries = Object.entries(score as Record) + .filter(([key]) => key.toLowerCase() !== 'discard') + .map(([key, val]) => [key, Number(val)] as const) + .filter(([, val]) => Number.isFinite(val)); + + if (numericEntries.length === 0) return null; + + const preferredMetric = + numericEntries.find(([key]) => key.toLowerCase() === 'score') ?? + numericEntries[0]; + const [firstMetric, firstValue] = preferredMetric; + + const tooltip = + numericEntries.length > 1 + ? numericEntries + .map(([metric, val]) => `${metric}: ${formatScoreValue(val)}`) + .join(' | ') + : undefined; + + return { + label: `${firstMetric}: ${formatScoreValue(firstValue)}`, + tooltip, + }; + } + + return null; + }; + const formatJobConfig = (job: any) => { const jobData = job?.job_data || {}; const interactiveType = @@ -176,6 +232,7 @@ const JobsList: React.FC = ({ const userInfo = jobData.user_info || {}; const userDisplay = userInfo.name || userInfo.email || ''; const providerDisplay = jobData.provider_name || job?.provider_name || ''; + const scoreDisplay = getScoreDisplay(jobData?.score); if (job?.placeholder) { return ( @@ -215,6 +272,21 @@ const JobsList: React.FC = ({ Title: {taskName} )} + {scoreDisplay && ( + + + {scoreDisplay.label} + + + )} ); } @@ -242,6 +314,21 @@ const JobsList: React.FC = ({ Provider: {providerDisplay} )} + {scoreDisplay && ( + + + {scoreDisplay.label} + + + )} ); } From f88ba059da2e7a5ce5a822f53f6d50b34e27c3b9 Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:41:18 -0600 Subject: [PATCH 4/8] format --- cli/src/transformerlab_cli/commands/job.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index 30a90aab71..e62b497711 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -254,9 +254,7 @@ def _score_items(score) -> list[tuple[str, object]]: return [] if isinstance(score, dict): return [ - (str(key), value) - for key, value in score.items() - if value is not None and str(key).lower() != "discard" + (str(key), value) for key, value in score.items() if value is not None and str(key).lower() != "discard" ] return [("score", score)] From e6d230e3720d85cd904db3d7313c7da9a010f4a1 Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:52:55 -0600 Subject: [PATCH 5/8] Fix race conditions and false bool evals --- api/transformerlab/routers/experiment/jobs.py | 31 ++++++++++++++----- api/transformerlab/services/job_service.py | 21 +++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/api/transformerlab/routers/experiment/jobs.py b/api/transformerlab/routers/experiment/jobs.py index e9f479f805..6a6c8de4a1 100644 --- a/api/transformerlab/routers/experiment/jobs.py +++ b/api/transformerlab/routers/experiment/jobs.py @@ -138,13 +138,30 @@ async def job_update_job_data(job_id: str, experimentId: str, body: dict = Body( # Keep discard under job_data.score.discard to avoid introducing a new top-level field. if "discard" in filtered: - discard_value = bool(filtered.pop("discard")) - existing_job = await job_service.job_get_cached(job_id, experiment_id=experimentId) - existing_job_data = (existing_job or {}).get("job_data", {}) if isinstance(existing_job, dict) else {} - existing_score = existing_job_data.get("score", {}) if isinstance(existing_job_data, dict) else {} - merged_score = dict(existing_score) if isinstance(existing_score, dict) else {} - merged_score["discard"] = discard_value - filtered["score"] = merged_score + raw_discard_value = filtered.pop("discard") + if isinstance(raw_discard_value, bool): + discard_value = raw_discard_value + elif isinstance(raw_discard_value, int): + if raw_discard_value not in (0, 1): + raise HTTPException(status_code=422, detail="discard must be a boolean value") + discard_value = bool(raw_discard_value) + elif isinstance(raw_discard_value, str): + normalized_discard_value = raw_discard_value.strip().lower() + if normalized_discard_value in {"true", "false"}: + discard_value = normalized_discard_value == "true" + elif normalized_discard_value: + try: + numeric_discard_value = int(normalized_discard_value) + except ValueError as exc: + raise HTTPException(status_code=422, detail="discard must be a boolean value") from exc + if numeric_discard_value not in (0, 1): + raise HTTPException(status_code=422, detail="discard must be a boolean value") + discard_value = bool(numeric_discard_value) + else: + raise HTTPException(status_code=422, detail="discard must be a boolean value") + else: + raise HTTPException(status_code=422, detail="discard must be a boolean value") + await job_service.job_update_job_data_score_field(job_id, "discard", discard_value, experimentId) if not filtered: return {"message": "No valid keys to update"} diff --git a/api/transformerlab/services/job_service.py b/api/transformerlab/services/job_service.py index 23a4e2d00b..cde9fd99fb 100644 --- a/api/transformerlab/services/job_service.py +++ b/api/transformerlab/services/job_service.py @@ -402,6 +402,27 @@ async def job_update_job_data_insert_key_values(job_id, updates: Dict[str, Any], print(f"Error updating job {job_id}: {e}") +async def job_update_job_data_score_field(job_id: str, score_key: str, value: Any, experiment_id: str) -> None: + """ + Merge a single key into job_data.score for a job. + """ + try: + resolved_id = await _resolve_full_job_id(str(job_id), str(experiment_id)) + actual_id = resolved_id or str(job_id) + + job = await Job.get(actual_id, experiment_id) + current = (await job.get_job_data()) or {} + score = current.get("score") if isinstance(current, dict) else {} + merged_score = dict(score) if isinstance(score, dict) else {} + merged_score[score_key] = value + await job.update_job_data_field("score", merged_score) + + await cache.invalidate(f"job:{actual_id}") + await cache.delete(_job_cache_key(actual_id)) + except Exception as e: + print(f"Error updating job {job_id} score field {score_key}: {e}") + + async def job_stop(job_id, experiment_id): print("Stopping job: " + str(job_id)) await job_update_job_data_insert_key_value(job_id, "stop", True, experiment_id) From c836b443ccac3aee31ae7c9714e1bdd3e49cfe2a Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 09:53:47 -0600 Subject: [PATCH 6/8] fix frontend conditions --- cli/src/transformerlab_cli/commands/job.py | 17 ++++++++- .../components/Experiment/Tasks/JobsList.tsx | 35 +++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index e62b497711..e5dd85a38c 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -275,7 +275,22 @@ def _is_discarded(job_data: dict) -> bool: score = job_data.get("score", {}) if not isinstance(score, dict): return False - return bool(score.get("discard", False)) + discard = score.get("discard", False) + if isinstance(discard, bool): + return discard + if isinstance(discard, int): + return discard == 1 + if isinstance(discard, str): + normalized = discard.strip().lower() + if normalized == "true": + return True + if normalized == "false": + return False + try: + return int(normalized) == 1 + except ValueError: + return False + return False def _render_jobs(jobs) -> Table: diff --git a/src/renderer/components/Experiment/Tasks/JobsList.tsx b/src/renderer/components/Experiment/Tasks/JobsList.tsx index f7f9ad02c7..ad7eadeddf 100644 --- a/src/renderer/components/Experiment/Tasks/JobsList.tsx +++ b/src/renderer/components/Experiment/Tasks/JobsList.tsx @@ -73,6 +73,33 @@ interface JobsListProps { onStopPendingChange?: (jobId: string, stopPending: boolean) => void; } +const parseDiscardValue = (value: unknown): boolean => { + if (typeof value === 'boolean') { + return value; + } + if (typeof value === 'number') { + if (value === 0 || value === 1) { + return Boolean(value); + } + return false; + } + if (typeof value === 'string') { + const normalized = value.trim().toLowerCase(); + if (normalized === 'true') { + return true; + } + if (normalized === 'false') { + return false; + } + const numeric = Number.parseInt(normalized, 10); + if (Number.isNaN(numeric)) { + return false; + } + return numeric === 1; + } + return false; +}; + const JobsList: React.FC = ({ jobs, launchProgressByJobId, @@ -701,11 +728,15 @@ const JobsList: React.FC = ({ onClick={() => onToggleDiscard?.( String(job.id), - !!job?.job_data?.score?.discard, + parseDiscardValue( + job?.job_data?.score?.discard, + ), ) } > - {job?.job_data?.score?.discard ? ( + {parseDiscardValue( + job?.job_data?.score?.discard, + ) ? ( <> Unmark discard From 607fb775d93be9941ea9c5dd0559d902c0ce8172 Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 10:13:36 -0600 Subject: [PATCH 7/8] Improve sorting filter and allow asc and desc order --- cli/src/transformerlab_cli/commands/job.py | 38 +++++++++++++++++----- cli/tests/commands/test_job.py | 25 +++++++++++--- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index e5dd85a38c..746e7daee1 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -427,7 +427,12 @@ def _render_job(job) -> None: console.print(panel) -def list_jobs(experiment_id: str, running_only: bool = False, sort_by: str | None = None): +def list_jobs( + experiment_id: str, + running_only: bool = False, + score_metric: str | None = None, + score_order: str = "desc", +): """List all jobs for a specific experiment.""" output_format = cli_state.output_format jobs = [] @@ -440,11 +445,11 @@ def list_jobs(experiment_id: str, running_only: bool = False, sort_by: str | Non if running_only: jobs = [j for j in jobs if j.get("status") in ACTIVE_JOB_STATUSES] - if sort_by: - # Sort by a score key (e.g. "eval/loss"). Jobs without the key go to the end. + if score_metric: + # Sort by score metric (e.g. "eval/loss"). Missing/non-numeric scores go to the end. def _sort_key(job): score = job.get("job_data", {}).get("score", {}) - val = score.get(sort_by) if isinstance(score, dict) else None + val = score.get(score_metric) if isinstance(score, dict) else None if val is None: return (1, 0.0) # push to end try: @@ -452,7 +457,7 @@ def _sort_key(job): except (ValueError, TypeError): return (1, 0.0) - jobs = sorted(jobs, key=_sort_key) + jobs = sorted(jobs, key=_sort_key, reverse=score_order == "desc") if output_format == "json": jobs = [{**job, "discarded": _is_discarded(job.get("job_data", {}))} for job in jobs] @@ -952,13 +957,30 @@ def command_job_list( running: bool = typer.Option( False, "--running", help="Show only active jobs (WAITING, LAUNCHING, RUNNING, INTERACTIVE)" ), - sort_by: str = typer.Option( - None, "--sort-by", help="Sort jobs by a score metric key (e.g. 'eval/loss'). Ascending order." + score_metric: str = typer.Option( + None, + "--score-metric", + "--sort-by", + help="Sort jobs by this score metric key (e.g. 'eval/loss').", + ), + score_order: str = typer.Option( + "desc", + "--score-order", + help="Score ordering direction: desc (default) or asc.", ), ): """List all jobs.""" + score_order_normalized = score_order.strip().lower() + if score_order_normalized not in {"desc", "asc"}: + console.print("[error]Error:[/error] --score-order must be either 'desc' or 'asc'.") + raise typer.Exit(1) current_experiment = require_current_experiment() - list_jobs(current_experiment, running_only=running, sort_by=sort_by) + list_jobs( + current_experiment, + running_only=running, + score_metric=score_metric, + score_order=score_order_normalized, + ) @app.command("info") diff --git a/cli/tests/commands/test_job.py b/cli/tests/commands/test_job.py index 791d6abf8a..66ab8e5087 100644 --- a/cli/tests/commands/test_job.py +++ b/cli/tests/commands/test_job.py @@ -239,20 +239,35 @@ def test_job_info_pretty_shows_score_without_discard(_mock_require, _mock_api): @patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) @patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") @patch("transformerlab_cli.commands.job.check_configs") -def test_job_list_sort_by_metric(_mock_check, _mock_require, _mock_api): - """Test that --sort-by sorts jobs by a score metric key in ascending order.""" - result = runner.invoke(app, ["--format", "json", "job", "list", "--sort-by", "eval/loss"]) +def test_job_list_sort_by_metric_desc_default(_mock_check, _mock_require, _mock_api): + """Test that score sorting defaults to descending and missing metric jobs are last.""" + result = runner.invoke(app, ["--format", "json", "job", "list", "--score-metric", "eval/loss"]) assert result.exit_code == 0 data = json.loads(result.output.strip()) - # Job 2 (eval/loss=2.1) should come before Job 4 (eval/loss=3.5), + # Job 4 (eval/loss=3.5) should come before Job 2 (eval/loss=2.1), # and jobs without the metric should be at the end. ids = [j["id"] for j in data] - assert ids.index(2) < ids.index(4) + assert ids.index(4) < ids.index(2) # Jobs without eval/loss (1, 3, 5) should be after jobs with it assert ids.index(2) < ids.index(1) assert ids.index(4) < ids.index(1) +@patch("transformerlab_cli.commands.job.api.get", return_value=_mock_api_response(SAMPLE_JOBS)) +@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1") +@patch("transformerlab_cli.commands.job.check_configs") +def test_job_list_sort_by_metric_asc(_mock_check, _mock_require, _mock_api): + """Test that score sorting can be switched to ascending.""" + result = runner.invoke( + app, + ["--format", "json", "job", "list", "--score-metric", "eval/loss", "--score-order", "asc"], + ) + assert result.exit_code == 0 + data = json.loads(result.output.strip()) + ids = [j["id"] for j in data] + assert ids.index(2) < ids.index(4) + + # --------------------------------------------------------------------------- # Info command tests # --------------------------------------------------------------------------- From 17eb83356ba25516c513ded555a3b9380fe9c6ee Mon Sep 17 00:00:00 2001 From: deep1401 Date: Mon, 4 May 2026 10:16:50 -0600 Subject: [PATCH 8/8] fix missing score jobs error --- cli/src/transformerlab_cli/commands/job.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cli/src/transformerlab_cli/commands/job.py b/cli/src/transformerlab_cli/commands/job.py index 746e7daee1..b4bed4b93c 100644 --- a/cli/src/transformerlab_cli/commands/job.py +++ b/cli/src/transformerlab_cli/commands/job.py @@ -446,18 +446,24 @@ def list_jobs( jobs = [j for j in jobs if j.get("status") in ACTIVE_JOB_STATUSES] if score_metric: - # Sort by score metric (e.g. "eval/loss"). Missing/non-numeric scores go to the end. - def _sort_key(job): + # Sort by score metric (e.g. "eval/loss"). + # Jobs with missing/non-numeric values are always appended at the end. + scored_jobs: list[tuple[dict, float]] = [] + missing_score_jobs: list[dict] = [] + + for job in jobs: score = job.get("job_data", {}).get("score", {}) - val = score.get(score_metric) if isinstance(score, dict) else None - if val is None: - return (1, 0.0) # push to end + raw_val = score.get(score_metric) if isinstance(score, dict) else None + if raw_val is None: + missing_score_jobs.append(job) + continue try: - return (0, float(val)) + scored_jobs.append((job, float(raw_val))) except (ValueError, TypeError): - return (1, 0.0) + missing_score_jobs.append(job) - jobs = sorted(jobs, key=_sort_key, reverse=score_order == "desc") + scored_jobs = sorted(scored_jobs, key=lambda item: item[1], reverse=score_order == "desc") + jobs = [job for job, _ in scored_jobs] + missing_score_jobs if output_format == "json": jobs = [{**job, "discarded": _is_discarded(job.get("job_data", {}))} for job in jobs]