Merge pull request #1932 from transformerlab/fix/job-logs-cli-messages

Fix error messages for CLI when logs are not available yet

Author: deep1401

api/transformerlab/routers/experiment/jobs.py

@@ -11,6 +11,7 @@
 from fastapi.responses import StreamingResponse, FileResponse
 from json import JSONDecodeError
 from lab import Job, storage
+from lab.job_status import JobStatus
 from sqlalchemy.ext.asyncio import AsyncSession
 from werkzeug.utils import secure_filename
 
@@ -69,6 +70,30 @@ def _eval_results_storage_path(stored_path: str, eval_dir: str) -> str:
     return p
 
 
+def _job_logs_not_ready_payload(
+    job_data: dict,
+    tail_lines: int,
+    message: str,
+    *,
+    include_request_id: bool = False,
+) -> dict:
+    payload = {
+        "cluster_name": job_data.get("cluster_name"),
+        "provider_id": job_data.get("provider_id"),
+        "provider_job_id": None,
+        "provider_name": job_data.get("provider_name"),
+        "tail_lines": tail_lines,
+        "logs": "",
+        "job_candidates": [],
+        "message": message,
+        "retryable": True,
+        "retry_after_seconds": 10,
+    }
+    if include_request_id:
+        payload["request_id"] = None
+    return payload
+
+
 @router.get("/list")
 async def jobs_get_all(
     experimentId: str,
@@ -271,15 +296,25 @@ async def get_provider_job_logs(
         raise HTTPException(status_code=404, detail="Job not found")
 
     job_data = job.get("job_data") or {}
+    job_status = str(job.get("status") or "").upper()
     if not isinstance(job_data, dict):
         try:
             job_data = json.loads(job_data)
         except JSONDecodeError:
             job_data = {}
 
+    def _logs_not_ready_response() -> dict:
+        return _job_logs_not_ready_payload(
+            job_data,
+            tail_lines,
+            "Machine logs are not available yet. Please try again shortly.",
+        )
+
     provider_id = job_data.get("provider_id")
     cluster_name = job_data.get("cluster_name")
     if not provider_id or not cluster_name:
+        if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+            return _logs_not_ready_response()
         raise HTTPException(
             status_code=400, detail="Job does not contain provider metadata (provider_id/cluster_name missing)"
         )
@@ -372,6 +407,8 @@ async def get_provider_job_logs(
             ]
 
     if provider_job_id is None:
+        if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+            return _logs_not_ready_response()
         raise HTTPException(status_code=404, detail="Unable to determine provider job id for this job")
 
     # For local provider, set workspace_dir (job dir) so LocalProvider can read logs.
@@ -437,6 +474,7 @@ async def get_request_logs(
         raise HTTPException(status_code=404, detail="Job not found")
 
     job_data = job.get("job_data") or {}
+    job_status = str(job.get("status") or "").upper()
     if not isinstance(job_data, dict):
         try:
             job_data = json.loads(job_data)
@@ -445,6 +483,13 @@ async def get_request_logs(
 
     provider_id = job_data.get("provider_id")
     if not provider_id:
+        if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+            return _job_logs_not_ready_payload(
+                job_data,
+                tail_lines,
+                "Request logs are not available yet. Please try again shortly.",
+                include_request_id=True,
+            )
         raise HTTPException(status_code=400, detail="Job does not contain provider metadata (provider_id missing)")
 
     provider_launch_result = job_data.get("provider_launch_result")
@@ -454,6 +499,13 @@ async def get_request_logs(
     if not request_id:
         request_id = job_data.get("orchestrator_request_id")
     if not request_id:
+        if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+            return _job_logs_not_ready_payload(
+                job_data,
+                tail_lines,
+                "Request logs are not available yet. Please try again shortly.",
+                include_request_id=True,
+            )
         raise HTTPException(status_code=400, detail="Job does not have a provider request ID")
 
     team_id = user_and_team["team_id"]
@@ -695,6 +747,7 @@ async def get_job_task_logs(
         raise HTTPException(status_code=404, detail="Job not found")
 
     job_data = job.get("job_data") or {}
+    job_status = str(job.get("status") or "").upper()
     if not isinstance(job_data, dict):
         try:
             job_data = json.loads(job_data)
@@ -711,9 +764,21 @@ async def get_job_task_logs(
         try:
             output_file_name = await shared.get_job_output_file_name(job_id, experiment_name=experimentId)
         except (ValueError, FileNotFoundError):
+            if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+                return _job_logs_not_ready_payload(
+                    job_data,
+                    tail_lines,
+                    "Task logs are not available yet. Please try again shortly.",
+                )
             return {"logs": "", "tail_lines": tail_lines}
 
     if not await storage.exists(output_file_name):
+        if job_status in {JobStatus.LAUNCHING.value, JobStatus.WAITING.value, "CREATED"}:
+            return _job_logs_not_ready_payload(
+                job_data,
+                tail_lines,
+                "Task logs are not available yet. Please try again shortly.",
+            )
         return {"logs": "", "tail_lines": tail_lines}
 
     async with await storage.open(output_file_name, "r") as f:

cli/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab-cli"
-version = "0.0.41"
+version = "0.0.42"
 description = "Transformer Lab CLI"
 requires-python = ">=3.10"
 authors = [{ name = "Transformer Lab", email = "hello@transformerlab.ai" }]

cli/src/transformerlab_cli/commands/job.py

@@ -760,16 +760,48 @@ def _print_logs(experiment_id: str, job_id: str, output_format: str, fetch_fn, l
         response = fetch_fn(experiment_id, job_id)
 
     if response.status_code != 200:
+        detail = ""
+        try:
+            payload = response.json()
+            if isinstance(payload, dict):
+                detail = str(payload.get("detail") or payload.get("message") or "")
+        except Exception:
+            detail = ""
+
         if output_format == "json":
-            print(json.dumps({"error": f"Failed to fetch {label}. Status code: {response.status_code}"}))
+            body = {"error": f"Failed to fetch {label}. Status code: {response.status_code}"}
+            if detail:
+                body["detail"] = detail
+            print(json.dumps(body))
         else:
             console.print(f"[red]Error:[/red] Failed to fetch {label}. Status code: {response.status_code}")
+            if detail:
+                console.print(f"[red]Detail:[/red] {detail}")
         raise typer.Exit(1)
 
     data = response.json()
     logs_text = data.get("logs", "") if isinstance(data, dict) else ""
+    wait_message = data.get("message", "") if isinstance(data, dict) else ""
+    retryable = bool(data.get("retryable")) if isinstance(data, dict) else False
 
     if not logs_text or "No log files found" in logs_text:
+        if wait_message and retryable:
+            if output_format == "json":
+                print(
+                    json.dumps(
+                        {
+                            "job_id": job_id,
+                            "logs": logs_text,
+                            "line_count": 0,
+                            "message": wait_message,
+                            "retryable": True,
+                            "retry_after_seconds": data.get("retry_after_seconds"),
+                        }
+                    )
+                )
+            else:
+                console.print(f"[yellow]{wait_message}[/yellow]")
+            return
         exit_with_no_results(output_format, f"No {label} found for this job")
 
     if output_format == "json":

cli/tests/commands/test_job.py

@@ -390,6 +390,46 @@ def test_machine_logs_error(_mock_require, _mock_fetch):
     assert result.exit_code == 1
 
 
+@patch(
+    "transformerlab_cli.commands.job.fetch_logs",
+    return_value=_mock_logs_response(""),
+)
+@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
+def test_machine_logs_not_ready_message_pretty(_mock_require, _mock_fetch):
+    """machine-logs shows retry message from API and exits cleanly when logs are not ready yet."""
+    _mock_fetch.return_value.json.return_value = {
+        "logs": "",
+        "message": "Machine logs are not available yet while the job is still launching. Please try again shortly.",
+        "retryable": True,
+        "retry_after_seconds": 10,
+    }
+    result = runner.invoke(app, ["job", "machine-logs", "42"])
+    assert result.exit_code == 0
+    assert "not available yet" in strip_ansi(result.output)
+
+
+@patch(
+    "transformerlab_cli.commands.job.fetch_logs",
+    return_value=_mock_logs_response(""),
+)
+@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
+def test_machine_logs_not_ready_message_json(_mock_require, _mock_fetch):
+    """machine-logs --format json preserves retry metadata when logs are not ready yet."""
+    _mock_fetch.return_value.json.return_value = {
+        "logs": "",
+        "message": "Machine logs are not available yet while the job is still launching. Please try again shortly.",
+        "retryable": True,
+        "retry_after_seconds": 10,
+    }
+    result = runner.invoke(app, ["--format", "json", "job", "machine-logs", "42"])
+    assert result.exit_code == 0
+    data = json.loads(result.output.strip())
+    assert data["job_id"] == "42"
+    assert data["line_count"] == 0
+    assert data["retryable"] is True
+    assert "not available yet" in data["message"]
+
+
 @patch(
     "transformerlab_cli.commands.job.fetch_task_logs",
     return_value=_mock_logs_response("sdk output line 1\nsdk output line 2"),
@@ -412,6 +452,24 @@ def test_task_logs_json(_mock_require, _mock_fetch):
     assert "sdk line" in data["logs"]
 
 
+@patch("transformerlab_cli.commands.job.fetch_task_logs", return_value=_mock_logs_response(""))
+@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
+def test_task_logs_not_ready_message_json(_mock_require, _mock_fetch):
+    """task-logs --format json preserves retry metadata when logs are not ready yet."""
+    _mock_fetch.return_value.json.return_value = {
+        "logs": "",
+        "message": "Task logs are not available yet while the job is still launching. Please try again shortly.",
+        "retryable": True,
+        "retry_after_seconds": 10,
+    }
+    result = runner.invoke(app, ["--format", "json", "job", "task-logs", "42"])
+    assert result.exit_code == 0
+    data = json.loads(result.output.strip())
+    assert data["job_id"] == "42"
+    assert data["retryable"] is True
+    assert "not available yet" in data["message"]
+
+
 @patch("transformerlab_cli.commands.job.api.get", return_value=_mock_logs_response("sdk line"))
 @patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
 def test_task_logs_hits_task_logs_endpoint(_mock_require, mock_get):
@@ -452,6 +510,24 @@ def test_request_logs_json(_mock_require, _mock_fetch):
     assert "launch log" in data["logs"]
 
 
+@patch("transformerlab_cli.commands.job.fetch_request_logs", return_value=_mock_logs_response(""))
+@patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
+def test_request_logs_not_ready_message_json(_mock_require, _mock_fetch):
+    """request-logs --format json preserves retry metadata when logs are not ready yet."""
+    _mock_fetch.return_value.json.return_value = {
+        "logs": "",
+        "message": "Request logs are not available yet while the job is still launching. Please try again shortly.",
+        "retryable": True,
+        "retry_after_seconds": 10,
+    }
+    result = runner.invoke(app, ["--format", "json", "job", "request-logs", "42"])
+    assert result.exit_code == 0
+    data = json.loads(result.output.strip())
+    assert data["job_id"] == "42"
+    assert data["retryable"] is True
+    assert "not available yet" in data["message"]
+
+
 @patch("transformerlab_cli.commands.job.fetch_request_logs", return_value=_mock_error_response(400))
 @patch("transformerlab_cli.commands.job.require_current_experiment", return_value="exp1")
 def test_request_logs_error(_mock_require, _mock_fetch):