nix-eval-jobs: add a 5m timeout

Author: Mic92

.github/eval.py

@@ -5,6 +5,7 @@
 """Evaluate flake outputs using nix-eval-jobs with index-based sharding."""
 
 import json
+import select
 import subprocess
 import sys
 import time
@@ -42,6 +43,9 @@
 """
 
 
+TIMEOUT_SECONDS = 300  # 5 minutes
+
+
 def run_eval(
     job_id: int, total_jobs: int
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
@@ -70,18 +74,35 @@ def run_eval(
     successes = []
     errors = []
     start_time = time.time()
+    timed_out = False
 
     with subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
     ) as proc:
         assert proc.stdout is not None
-        for line in proc.stdout:
+        while True:
+            elapsed = time.time() - start_time
+            if elapsed > TIMEOUT_SECONDS:
+                print(f"\n[{elapsed:6.1f}s] ⏱ Timeout after {TIMEOUT_SECONDS}s")
+                proc.terminate()
+                timed_out = True
+                break
+
+            # Use select to avoid blocking indefinitely
+            readable, _, _ = select.select([proc.stdout], [], [], 1.0)
+            if not readable:
+                continue
+
+            line = proc.stdout.readline()
+            if not line:
+                break  # EOF
+
             line = line.strip()
             if not line:
                 continue
+
             try:
                 result = json.loads(line)
-                elapsed = time.time() - start_time
                 attr = result.get("attr", "?")
                 if "error" in result:
                     errors.append(result)
@@ -98,6 +119,9 @@ def run_eval(
 
         proc.wait()
 
+    if timed_out:
+        errors.append({"attr": "timeout", "error": f"Evaluation timed out after {TIMEOUT_SECONDS}s"})
+
     return successes, errors