Merge pull request #1376 from transformerlab/fix/make-provider-logs-permanent

Make provider logs permanent

Author: deep1401

api/pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
   "soundfile==0.13.1",
   "tensorboardX==2.6.2.2",
   "timm==1.0.15",
-  "transformerlab==0.0.81",
+  "transformerlab==0.0.82",
   "transformerlab-inference==0.2.52",
   "transformers==4.57.1",
   "wandb==0.23.1",

api/transformerlab/routers/experiment/jobs.py

@@ -241,11 +241,22 @@ async def get_provider_job_logs(
     experimentId: str,
     job_id: str,
     tail_lines: int = Query(400, ge=100, le=2000),
+    live: bool = Query(
+        False,
+        description=(
+            "If true, bypass cached provider_logs.txt and fetch logs directly from the underlying compute provider."
+        ),
+    ),
     user_and_team=Depends(get_user_and_team),
     session: AsyncSession = Depends(get_async_session),
 ):
     """
     Fetch the raw job logs directly from the underlying compute provider for a REMOTE job.
+
+    Preferred order:
+      1. If `provider_logs.txt` exists in the job directory (written by the SDK wrapper),
+         read and return that.
+      2. Otherwise, fall back to provider-native log retrieval (existing behavior).
     """
 
     job = await job_service.job_get(job_id)
@@ -266,6 +277,36 @@ async def get_provider_job_logs(
             status_code=400, detail="Job does not contain provider metadata (provider_id/cluster_name missing)"
         )
 
+    # 1) If live=False (default), first try to read provider logs from the job directory
+    #    via the SDK's job_dir helper. This file is written by tfl-remote-trap inside
+    #    the remote environment.
+    if not live:
+        try:
+            from lab.dirs import get_job_dir
+
+            job_dir = await get_job_dir(job_id)
+            provider_logs_path = storage.join(job_dir, "provider_logs.txt")
+            if await storage.exists(provider_logs_path):
+                async with await storage.open(provider_logs_path, "r", encoding="utf-8") as f:
+                    logs_text = await f.read()
+                if tail_lines is not None:
+                    lines = logs_text.splitlines()
+                    logs_text = "\n".join(lines[-tail_lines:])
+
+                return {
+                    "cluster_name": cluster_name,
+                    "provider_id": provider_id,
+                    "provider_job_id": None,
+                    "provider_name": job_data.get("provider_name"),
+                    "tail_lines": tail_lines,
+                    "logs": logs_text,
+                    "job_candidates": [],
+                }
+        except Exception:
+            # If anything goes wrong with the file-based path, fall back to provider-native logs.
+            pass
+
+    # 2) Fall back to existing provider-native log retrieval.
     provider = await get_team_provider(session, user_and_team["team_id"], provider_id)
     if not provider:
         raise HTTPException(status_code=404, detail="Provider not found")

lab-sdk/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab"
-version = "0.0.81"
+version = "0.0.82"
 description = "Python SDK for Transformer Lab"
 readme = "README.md"
 requires-python = ">=3.10"

lab-sdk/src/lab/remote_trap.py

@@ -6,7 +6,7 @@
 import sys
 from typing import List
 
-from lab import Job
+from lab import Job, storage
 
 
 async def _set_live_status_async(job_id: str, status: str) -> None:
@@ -43,6 +43,54 @@ def _set_live_status(status: str) -> None:
             return
 
 
+async def _write_provider_logs_async(job_id: str, logs_text: str) -> None:
+    """
+    Best-effort helper to write combined stdout/stderr logs to the job directory.
+
+    Uses _TFL_JOB_ID to resolve the job directory via lab.dirs.get_job_dir, then
+    writes provider_logs.txt using the storage abstraction.
+    """
+    try:
+        # Import inside helper to avoid circular imports at module load time.
+        from lab.dirs import get_job_dir
+
+        job_dir = await get_job_dir(job_id)
+        log_path = storage.join(job_dir, "provider_logs.txt")
+
+        # Ensure the directory exists (no-op for remote storage that doesn't require mkdirs).
+        try:
+            await storage.makedirs(job_dir, exist_ok=True)
+        except Exception:
+            # Some storage backends may not support makedirs for virtual paths; ignore.
+            pass
+
+        async with await storage.open(log_path, "w", encoding="utf-8") as f:
+            await f.write(logs_text or "")
+    except Exception:
+        # Never let logging failures break the wrapped command.
+        return
+
+
+def _write_provider_logs(logs_text: str) -> None:
+    """Entry-point wrapper for writing provider logs for the current job."""
+    job_id = os.environ.get("_TFL_JOB_ID")
+    if not job_id or logs_text is None:
+        return
+
+    try:
+        asyncio.run(_write_provider_logs_async(job_id, logs_text))
+    except RuntimeError:
+        # Fallback in case an event loop already exists.
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                loop.create_task(_write_provider_logs_async(job_id, logs_text))
+            else:
+                loop.run_until_complete(_write_provider_logs_async(job_id, logs_text))
+        except Exception:
+            return
+
+
 def main(argv: List[str] | None = None) -> int:
     """
     Wrapper entrypoint for remote jobs.
@@ -74,7 +122,40 @@ def main(argv: List[str] | None = None) -> int:
     _set_live_status("started")
 
     # Run the original command in the shell so it behaves exactly as submitted.
-    completed = subprocess.run(command_str, shell=True)
+    # Capture stdout/stderr so we can save a copy to provider_logs.txt while still
+    # echoing output to the current process streams.
+    completed = subprocess.run(
+        command_str,
+        shell=True,
+        capture_output=True,
+        text=True,
+    )
+
+    # Echo captured output back to the current stdout/stderr so provider-native logs
+    # (e.g., SkyPilot, SLURM, RunPod) still see the same content.
+    if completed.stdout:
+        try:
+            sys.stdout.write(completed.stdout)
+            sys.stdout.flush()
+        except Exception:
+            pass
+    if completed.stderr:
+        try:
+            sys.stderr.write(completed.stderr)
+            sys.stderr.flush()
+        except Exception:
+            pass
+
+    # Combine stdout + stderr into a single text blob and store it alongside the job.
+    combined_logs_parts: List[str] = []
+    if completed.stdout:
+        combined_logs_parts.append(completed.stdout)
+    if completed.stderr:
+        combined_logs_parts.append(completed.stderr)
+    combined_logs = "\n".join(part.rstrip("\n") for part in combined_logs_parts)
+
+    _write_provider_logs(combined_logs)
+
     exit_code = completed.returncode
 
     # Update live_status based on outcome (best-effort).

src/renderer/components/Experiment/Tasks/ViewOutputModalStreaming.tsx

@@ -2,6 +2,7 @@ import React, { useEffect, useMemo, useRef, useState } from 'react';
 import {
   Alert,
   Box,
+  Checkbox,
   CircularProgress,
   Modal,
   ModalClose,
@@ -117,6 +118,8 @@ function ViewOutputModalStreaming({
 }: ViewOutputModalStreamingProps) {
   const { experimentInfo } = useExperimentInfo();
   const [activeTab, setActiveTab] = useState<'output' | 'provider'>('output');
+  const [viewLiveProviderLogs, setViewLiveProviderLogs] =
+    useState<boolean>(false);
 
   const tabs = tabsProp.length > 0 ? tabsProp : ['output', 'provider'];
   const showTabList = tabs.length > 1;
@@ -128,6 +131,7 @@ function ViewOutputModalStreaming({
         ? current
         : ((tabs[0] ?? 'output') as 'output' | 'provider'),
     );
+    setViewLiveProviderLogs(false);
     // tabsKey is a stable serialization of tabs to avoid array reference churn
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [jobId, tabsKey]);
@@ -139,8 +143,10 @@ function ViewOutputModalStreaming({
     return chatAPI.Endpoints.Experiment.GetProviderLogs(
       experimentInfo.id,
       String(jobId),
+      400,
+      viewLiveProviderLogs,
     );
-  }, [experimentInfo?.id, jobId]);
+  }, [experimentInfo?.id, jobId, viewLiveProviderLogs]);
 
   const {
     data: providerLogsData,
@@ -210,29 +216,53 @@ function ViewOutputModalStreaming({
             </TabList>
           </Tabs>
         )}
+        {activeTab === 'provider' && (
+          <Box
+            sx={{
+              mt: 1,
+              display: 'flex',
+              alignItems: 'center',
+              gap: 1.5,
+            }}
+          >
+            <Checkbox
+              size="sm"
+              checked={viewLiveProviderLogs}
+              onChange={(event) =>
+                setViewLiveProviderLogs(!!event.target.checked)
+              }
+              label="View live provider logs"
+            />
+            {viewLiveProviderLogs && (
+              <Typography level="body-xs" color="warning">
+                Live logs are fetched directly from the remote machine and may
+                disappear once the machine stops running.
+              </Typography>
+            )}
+          </Box>
+        )}
         <Box
           sx={{
-            mt: 1,
+            mt: activeTab === 'provider' ? 0.5 : 1,
             flex: 1,
             minHeight: 0,
             width: '100%',
             display: 'flex',
-            padding: '8px 0px 8px 11px',
-            backgroundColor: '#000',
-            borderRadius: '8px',
+            flexDirection: 'column',
           }}
         >
           {tabs.includes('output') && activeTab === 'output' ? (
             <Box
               sx={{
-                padding: `0`,
+                padding: 0,
                 backgroundColor: '#000',
                 width: '100%',
                 flex: 1,
                 minHeight: 0,
                 overflow: 'hidden',
                 display: 'flex',
                 flexDirection: 'column',
+                borderRadius: '8px',
               }}
             >
               <PollingOutputTerminal
@@ -244,7 +274,19 @@ function ViewOutputModalStreaming({
               />
             </Box>
           ) : (
-            <>
+            <Box
+              sx={{
+                flex: 1,
+                minHeight: 0,
+                width: '100%',
+                display: 'flex',
+                flexDirection: 'column',
+                backgroundColor: '#000',
+                borderRadius: '8px',
+                padding: '8px 11px',
+                gap: 1,
+              }}
+            >
               {providerLogsLoading && (
                 <Box
                   sx={{
@@ -302,7 +344,7 @@ function ViewOutputModalStreaming({
               {!providerLogsLoading && !providerLogsError && (
                 <ProviderLogsTerminal logsText={providerLogText} />
               )}
-            </>
+            </Box>
           )}
         </Box>
       </ModalDialog>

src/renderer/lib/api-client/endpoints.ts

@@ -540,8 +540,9 @@ Endpoints.Experiment = {
     experimentId: string,
     jobId: string,
     tailLines: number = 400,
+    live: boolean = false,
   ) =>
-    `${API_URL()}experiment/${experimentId}/jobs/${jobId}/provider_logs?tail_lines=${tailLines}`,
+    `${API_URL()}experiment/${experimentId}/jobs/${jobId}/provider_logs?tail_lines=${tailLines}&live=${live}`,
   GetTunnelInfo: (
     experimentId: string,
     jobId: string,