Skip to content

Commit 6869fa9

Browse files
Mic92brianmcgillion
Mic92
authored and
brianmcgillion
committed
ci/eval: rewrite script to use nix-eval-jobs --select
This way we make no longer need impure evaluation. Also improved the error output so it looks like this: [ 24.3s] ✓ packages.aarch64-linux.hardware-scan [ 32.9s] ✓ packages.aarch64-linux.nvidia-jetson-orin-agx-release-nodemoapps [ 36.2s] ✓ packages.aarch64-linux.nxp-imx8mp-evk-release [ 37.7s] ✓ packages.x86_64-linux.audit-rules [ 37.9s] ✗ packages.x86_64-linux.demo-tower-mk1-debug-installer [ 38.0s] ✓ packages.x86_64-linux.ghaf-vms [ 38.2s] ✗ packages.x86_64-linux.lenovo-t14-amd-gen5-release-installer [ 38.4s] ✗ packages.x86_64-linux.lenovo-x1-carbon-gen11-release-installer [ 38.6s] ✗ packages.x86_64-linux.lenovo-x1-extras-debug-installer [ 40.6s] ✓ packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64-flash-qspi [ 48.8s] ✓ packages.x86_64-linux.nvidia-jetson-orin-agx-industrial-debug-nodemoapps-from-x86_64-flash-script [ 58.5s] ✓ packages.x86_64-linux.nvidia-jetson-orin-agx-release-nodemoapps-from-x86_64 [ 60.5s] ✓ packages.x86_64-linux.nvidia-jetson-orin-agx64-release-from-x86_64-flash-qspi [ 68.7s] ✓ packages.x86_64-linux.nvidia-jetson-orin-nx-debug-nodemoapps-from-x86_64-flash-script [ 69.7s] ✓ packages.x86_64-linux.rtl8126 [ 79.1s] ✓ packages.x86_64-linux.vm-debug ============================================================ Evaluated 16 attributes in 79.2s ✓ 12 succeeded ✗ 4 failed ============================================================ Errors: packages.x86_64-linux.demo-tower-mk1-debug-installer: error: … from call site at «github:tiiuae/nixpkgs/f5588cc02080b7fcc2c47b5d7daf44fb3c5dd476?narHash=sha256-oOmNB5vpk3y%2B7DbGzYPwFwuDORqtJq5NW2BBBZi0Vcg%3D»/lib/attrsets.nix:1344:61: 1343| */ 1344| genAttrs = names: f: genAttrs' names (n: nameValuePair n (f n)); | ^ 1345| The script is now overall a bit simpler what we had before Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
1 parent 9405925 commit 6869fa9

File tree

3 files changed

+164
-199
lines changed

3 files changed

+164
-199
lines changed

‎.github/eval.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#!/usr/bin/env python3
2+
# SPDX-FileCopyrightText: 2022-2026 TII (SSRC) and the Ghaf contributors
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Evaluate flake outputs using nix-eval-jobs with index-based sharding."""
6+
7+
import json
8+
import subprocess
9+
import sys
10+
import time
11+
from typing import Any
12+
13+
SELECT_EXPR = """
14+
flake: let
15+
lib = flake.inputs.nixpkgs.lib;
16+
jobId = {job_id};
17+
totalJobs = {total_jobs};
18+
19+
# Shard an attrset: keep attrs where (globalIdx + localIdx) mod totalJobs == jobId
20+
shardAttrs = globalIdx: attrs:
21+
let
22+
names = builtins.attrNames attrs;
23+
selected = lib.imap0 (i: name:
24+
if lib.mod (globalIdx + i) totalJobs == jobId then name else null
25+
) names;
26+
in lib.getAttrs (builtins.filter (x: x != null) selected) attrs;
27+
28+
# Apply sharding across all systems for an output type
29+
shardOutput = outputName:
30+
let
31+
output = flake.${{outputName}} or {{}};
32+
systems = builtins.attrNames output;
33+
offsets = builtins.foldl' (acc: sys:
34+
acc // {{ ${{sys}} = acc._idx; _idx = acc._idx + builtins.length (builtins.attrNames output.${{sys}}); }}
35+
) {{ _idx = 0; }} systems;
36+
in builtins.mapAttrs (sys: attrs: shardAttrs offsets.${{sys}} attrs) output;
37+
38+
in {{
39+
packages = shardOutput "packages";
40+
devShells = shardOutput "devShells";
41+
}}
42+
"""
43+
44+
45+
def run_eval(
46+
job_id: int, total_jobs: int
47+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
48+
"""Run nix-eval-jobs and return (successes, errors)."""
49+
select_expr = SELECT_EXPR.format(job_id=job_id, total_jobs=total_jobs)
50+
51+
cmd = [
52+
"nix",
53+
"run",
54+
"--inputs-from",
55+
".#",
56+
"nixpkgs#nix-eval-jobs",
57+
"--",
58+
"--flake",
59+
".#",
60+
"--no-instantiate",
61+
"--select",
62+
select_expr,
63+
"--force-recurse",
64+
"--accept-flake-config",
65+
"--option",
66+
"allow-import-from-derivation",
67+
"false",
68+
]
69+
70+
successes = []
71+
errors = []
72+
start_time = time.time()
73+
74+
print("[+] Starting nix-eval-jobs...", flush=True)
75+
76+
with subprocess.Popen(
77+
cmd, stdout=subprocess.PIPE, text=True
78+
) as proc:
79+
assert proc.stdout is not None
80+
for line in proc.stdout:
81+
line = line.strip()
82+
if not line:
83+
continue
84+
try:
85+
result = json.loads(line)
86+
elapsed = time.time() - start_time
87+
attr = result.get("attr", "?")
88+
if "error" in result:
89+
errors.append(result)
90+
print(f"[{elapsed:6.1f}s] ✗ {attr}")
91+
else:
92+
successes.append(result)
93+
print(f"[{elapsed:6.1f}s] ✓ {attr}")
94+
except json.JSONDecodeError:
95+
# Not JSON, probably a warning
96+
print(line, file=sys.stderr)
97+
98+
exit_code = proc.wait()
99+
if exit_code != 0 and not errors:
100+
errors.append({"attr": "nix-eval-jobs", "error": f"Process exited with code {exit_code}"})
101+
102+
return successes, errors
103+
104+
105+
def print_results(
106+
successes: list[dict[str, Any]], errors: list[dict[str, Any]], elapsed: float
107+
) -> None:
108+
"""Pretty print evaluation results."""
109+
print(f"\n{'=' * 60}")
110+
print(f"Evaluated {len(successes) + len(errors)} attributes in {elapsed:.1f}s")
111+
print(f" ✓ {len(successes)} succeeded")
112+
print(f" ✗ {len(errors)} failed")
113+
114+
if errors:
115+
print(f"\n{'=' * 60}")
116+
print("Errors:\n")
117+
for err in errors:
118+
attr = err.get("attr", "unknown")
119+
error_msg = err.get("error", "unknown error")
120+
print(f" {attr}:")
121+
# Indent error message
122+
for line in error_msg.split("\n"):
123+
print(f" {line}")
124+
print()
125+
126+
127+
def main() -> int:
128+
if len(sys.argv) != 3:
129+
print(f"Usage: {sys.argv[0]} <job-id> <total-jobs>", file=sys.stderr)
130+
return 1
131+
132+
try:
133+
job_id = int(sys.argv[1])
134+
total_jobs = int(sys.argv[2])
135+
except ValueError:
136+
print("Error: job-id and total-jobs must be integers", file=sys.stderr)
137+
return 1
138+
139+
if job_id < 0 or total_jobs <= 0 or job_id >= total_jobs:
140+
print("Error: invalid job-id or total-jobs", file=sys.stderr)
141+
return 1
142+
143+
print(f"[+] Evaluating flake outputs (job {job_id}/{total_jobs})")
144+
145+
start_time = time.time()
146+
successes, errors = run_eval(job_id, total_jobs)
147+
elapsed = time.time() - start_time
148+
print_results(successes, errors, elapsed)
149+
sys.stdout.flush()
150+
151+
return 1 if errors else 0
152+
153+
154+
if __name__ == "__main__":
155+
sys.exit(main())

‎.github/eval.sh‎

Lines changed: 0 additions & 181 deletions
This file was deleted.

‎.github/workflows/eval.yml‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,11 @@ permissions:
1616
jobs:
1717
eval:
1818
runs-on: ubuntu-latest
19-
timeout-minutes: 360
19+
timeout-minutes: 120
2020
strategy:
21+
fail-fast: false
2122
matrix:
22-
include:
23-
- jobid: 0
24-
- jobid: 1
25-
- jobid: 2
26-
- jobid: 3
27-
- jobid: 4
28-
- jobid: 5
29-
- jobid: 6
30-
- jobid: 7
31-
- jobid: 8
32-
- jobid: 9
23+
jobid: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
3324
concurrency:
3425
# Cancel any in-progress workflow runs from the same PR or branch,
3526
# allowing matrix jobs to run concurrently:
@@ -47,13 +38,13 @@ jobs:
4738
ref: ${{ github.event.pull_request.merge.sha || github.ref }}
4839
fetch-depth: 0
4940
persist-credentials: false
41+
5042
- name: Install nix
5143
uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
52-
- name: Evaluate (jobid=${{ matrix.jobid }})
53-
run: |
54-
echo "JOB_TOTAL: ${JOB_TOTAL}"
55-
echo "matrix.jobid: ${{ matrix.jobid }}"
56-
nix shell nixpkgs#nix-eval-jobs nixpkgs#jq \
57-
--command .github/eval.sh -t '(devShells\.|packages\.)' -j ${{ matrix.jobid }} -m "${JOB_TOTAL}"
44+
45+
- name: Evaluate (job ${{ matrix.jobid }}/${{ strategy.job-total }})
5846
env:
47+
JOB_ID: ${{ matrix.jobid }}
5948
JOB_TOTAL: ${{ strategy.job-total }}
49+
run: |
50+
python3 .github/eval.py "$JOB_ID" "$JOB_TOTAL"

0 commit comments

Comments
 (0)