Progress

cli.py

@@ -3,6 +3,8 @@
 
 import argparse
 import os
+import sys
+import time
 import webbrowser
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -24,6 +26,37 @@ def _projects(args) -> str:
     )
 
 
+def _progress_printer():
+    """Single-line stderr progress. Throttled to ~5 updates/sec so big scans
+    don't spam the terminal but still prove the process is alive."""
+    state = {"last": 0.0}
+    is_tty = sys.stderr.isatty()
+
+    def cb(i, total, path, totals):
+        now = time.monotonic()
+        final = (i == total)
+        if not final and (now - state["last"] < 0.2):
+            return
+        state["last"] = now
+        name = path.name
+        if len(name) > 48:
+            name = name[:45] + "..."
+        line = (
+            f"scanning {i}/{total}  "
+            f"files={totals['files']} msgs={totals['messages']} tools={totals['tools']}  "
+            f"{name}"
+        )
+        if is_tty:
+            sys.stderr.write("\r\x1b[2K" + line)
+            if final:
+                sys.stderr.write("\n")
+        else:
+            sys.stderr.write(line + "\n")
+        sys.stderr.flush()
+
+    return cb
+
+
 def _today_range():
     now = datetime.now(timezone.utc)
     start = datetime(now.year, now.month, now.day, tzinfo=timezone.utc).isoformat()
@@ -34,7 +67,7 @@ def _today_range():
 def cmd_scan(args):
     db = _db_path(args)
     init_db(db)
-    n = scan_dir(_projects(args), db)
+    n = scan_dir(_projects(args), db, progress=_progress_printer())
     print(f"Token Dashboard: scanned {n['files']} files, {n['messages']} messages, {n['tools']} tool calls")
 
 
@@ -74,7 +107,7 @@ def cmd_dashboard(args):
     db = _db_path(args)
     init_db(db)
     if not args.no_scan:
-        scan_dir(_projects(args), db)
+        scan_dir(_projects(args), db, progress=_progress_printer())
     from token_dashboard.server import run
 
     host = os.environ.get("HOST", "127.0.0.1")

token_dashboard/scanner.py

@@ -4,7 +4,7 @@
 import json
 import time
 from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import Callable, List, Optional, Tuple, Union
 
 from .db import connect
 
@@ -242,22 +242,37 @@ def scan_file(path: Path, project_slug: str, conn, start_byte: int = 0) -> dict:
     return {"messages": msgs, "tools": tools, "end_offset": end_offset}
 
 
-def scan_dir(projects_root: Union[str, Path], db_path: Union[str, Path]) -> dict:
+def scan_dir(
+    projects_root: Union[str, Path],
+    db_path: Union[str, Path],
+    progress: Optional[Callable[[int, int, Path, dict], None]] = None,
+) -> dict:
+    """Incrementally scan JSONL transcripts under ``projects_root`` into the DB.
+
+    ``progress`` is called as ``progress(index, total, path, totals)`` after
+    each file is processed (skipped or scanned). ``index`` is 1-based.
+    """
     root = Path(projects_root)
     totals = {"messages": 0, "tools": 0, "files": 0}
     if not root.is_dir():
         return totals
+    paths = list(root.rglob("*.jsonl"))
+    total = len(paths)
     with connect(db_path) as conn:
-        for p in root.rglob("*.jsonl"):
+        for i, p in enumerate(paths, start=1):
             try:
                 stat = p.stat()
             except OSError:
+                if progress:
+                    progress(i, total, p, totals)
                 continue
             row = conn.execute(
                 "SELECT mtime, bytes_read FROM files WHERE path=?", (str(p),)
             ).fetchone()
             offset = 0
             if row and row["mtime"] == stat.st_mtime and row["bytes_read"] == stat.st_size:
+                if progress:
+                    progress(i, total, p, totals)
                 continue
             if row and stat.st_size > row["bytes_read"]:
                 offset = row["bytes_read"]
@@ -273,5 +288,7 @@ def scan_dir(projects_root: Union[str, Path], db_path: Union[str, Path]) -> dict
             totals["messages"] += sub["messages"]
             totals["tools"]    += sub["tools"]
             totals["files"]    += 1
+            if progress:
+                progress(i, total, p, totals)
         conn.commit()
     return totals