|
| 1 | +"""Generate requirements lockfiles with multi-arch hashes from the AIPCC index. |
| 2 | +
|
| 3 | +Resolves dependencies with uv pip compile, then fetches per-architecture |
| 4 | +wheel hashes directly from the AIPCC simple index API. No Docker required. |
| 5 | +
|
| 6 | +Usage: |
| 7 | + python scripts/compile_requirements.py |
| 8 | +""" |
| 9 | + |
| 10 | +import re |
| 11 | +import subprocess |
| 12 | +import sys |
| 13 | +import urllib.request |
| 14 | +from pathlib import Path |
| 15 | + |
| 16 | +AIPCC_INDEX_URL = "https://console.redhat.com/api/pypi/public-rhai/rhoai/3.4/cpu-ubi9/simple" |
| 17 | + |
| 18 | +PYTHON_VERSION = "3.12" |
| 19 | + |
| 20 | +ARCHES = ["x86_64", "aarch64", "ppc64le", "s390x"] |
| 21 | + |
| 22 | +REPO_ROOT = Path(__file__).resolve().parent.parent |
| 23 | + |
| 24 | + |
| 25 | +def _parse_build_requires() -> list[str]: |
| 26 | + """Extract [build-system] requires from pyproject.toml.""" |
| 27 | + content = (REPO_ROOT / "pyproject.toml").read_text() |
| 28 | + match = re.search(r"\[build-system\].*?requires\s*=\s*\[(.*?)\]", content, re.DOTALL) |
| 29 | + if not match: |
| 30 | + raise SystemExit("Could not find [build-system] requires in pyproject.toml") |
| 31 | + return [dep.strip().strip("\"'") for dep in match.group(1).split(",") if dep.strip().strip("\"'")] |
| 32 | + |
| 33 | + |
| 34 | +def log(msg: str) -> None: |
| 35 | + """Print a progress message to stderr.""" |
| 36 | + print(msg, file=sys.stderr, flush=True) |
| 37 | + |
| 38 | + |
| 39 | +def _canonicalize(name: str) -> str: |
| 40 | + return re.sub(r"[-_.]+", "-", name).lower() |
| 41 | + |
| 42 | + |
| 43 | +def resolve_packages( |
| 44 | + in_files: list[str] | None = None, |
| 45 | + packages: list[str] | None = None, |
| 46 | + extra_args: list[str] | None = None, |
| 47 | +) -> list[tuple[str, str, str]]: |
| 48 | + """Run uv pip compile to resolve package names and versions. |
| 49 | +
|
| 50 | + Either in_files (paths relative to REPO_ROOT) or packages (list of |
| 51 | + package specifiers) must be provided. |
| 52 | + """ |
| 53 | + extra_args = extra_args or [] |
| 54 | + cmd = [ |
| 55 | + "uv", |
| 56 | + "pip", |
| 57 | + "compile", |
| 58 | + "--no-header", |
| 59 | + "--no-annotate", |
| 60 | + "--python-version", |
| 61 | + PYTHON_VERSION, |
| 62 | + "--index-url", |
| 63 | + AIPCC_INDEX_URL, |
| 64 | + *extra_args, |
| 65 | + ] |
| 66 | + |
| 67 | + stdin_input = None |
| 68 | + if in_files: |
| 69 | + cmd.extend(str(REPO_ROOT / f) for f in in_files) |
| 70 | + elif packages: |
| 71 | + stdin_input = "\n".join(packages) |
| 72 | + cmd.append("-") |
| 73 | + |
| 74 | + log(f" running: {' '.join(cmd)}") |
| 75 | + try: |
| 76 | + result = subprocess.run( |
| 77 | + cmd, |
| 78 | + input=stdin_input, |
| 79 | + capture_output=True, |
| 80 | + text=True, |
| 81 | + cwd=REPO_ROOT, |
| 82 | + timeout=120, |
| 83 | + ) |
| 84 | + except subprocess.TimeoutExpired: |
| 85 | + raise SystemExit("uv pip compile timed out after 120s. Check AIPCC index connectivity.") |
| 86 | + if result.returncode != 0: |
| 87 | + log(result.stderr) |
| 88 | + raise SystemExit(f"uv pip compile failed (exit {result.returncode}).") |
| 89 | + |
| 90 | + resolved = [] |
| 91 | + for line in result.stdout.splitlines(): |
| 92 | + line = line.strip() |
| 93 | + if not line or line.startswith("#") or line.startswith("--"): |
| 94 | + continue |
| 95 | + m = re.match(r"([a-zA-Z0-9_.-]+)==([^\s;]+)\s*(;.*)?", line) |
| 96 | + if m: |
| 97 | + resolved.append((m.group(1), m.group(2), (m.group(3) or "").strip())) |
| 98 | + |
| 99 | + return resolved |
| 100 | + |
| 101 | + |
| 102 | +def fetch_hashes_from_index(name: str, version: str) -> list[str]: |
| 103 | + """Fetch wheel hashes for all target architectures from the AIPCC simple index.""" |
| 104 | + canon = _canonicalize(name) |
| 105 | + url = f"{AIPCC_INDEX_URL}/{canon}/" |
| 106 | + |
| 107 | + try: |
| 108 | + with urllib.request.urlopen(url, timeout=30) as resp: |
| 109 | + html = resp.read().decode() |
| 110 | + except Exception as e: |
| 111 | + log(f" WARNING: could not fetch index for {name}: {e}") |
| 112 | + return [] |
| 113 | + |
| 114 | + hashes = [] |
| 115 | + version_escaped = re.escape(version) |
| 116 | + name_pattern = re.sub(r"[-_.]", "[-_.]", canon) |
| 117 | + |
| 118 | + for match in re.finditer( |
| 119 | + rf'href="[^"]*({name_pattern}-{version_escaped}(?:-\d+)?-[^"]*\.whl)#sha256=([a-f0-9]+)"', |
| 120 | + html, |
| 121 | + re.IGNORECASE, |
| 122 | + ): |
| 123 | + wheel_name = match.group(1) |
| 124 | + sha = match.group(2) |
| 125 | + |
| 126 | + is_any = "none-any" in wheel_name |
| 127 | + is_target_arch = any(arch in wheel_name for arch in ARCHES) |
| 128 | + is_abi3 = "abi3" in wheel_name |
| 129 | + is_cp = f"cp{PYTHON_VERSION.replace('.', '')}" in wheel_name |
| 130 | + |
| 131 | + if is_any or (is_target_arch and (is_abi3 or is_cp)): |
| 132 | + hashes.append(f"sha256:{sha}") |
| 133 | + |
| 134 | + return sorted(set(hashes)) |
| 135 | + |
| 136 | + |
| 137 | +def write_lockfile(out_file: str, resolved: list[tuple[str, str, str]]) -> None: |
| 138 | + """Fetch multi-arch hashes and write the lockfile.""" |
| 139 | + lines = [ |
| 140 | + f"--index-url {AIPCC_INDEX_URL}", |
| 141 | + "", |
| 142 | + ] |
| 143 | + |
| 144 | + for name, version, marker in resolved: |
| 145 | + hashes = fetch_hashes_from_index(name, version) |
| 146 | + marker_part = f" {marker}" if marker else "" |
| 147 | + |
| 148 | + if not hashes: |
| 149 | + log(f" WARNING: no matching wheels found for {name}=={version}") |
| 150 | + lines.append(f"{name}=={version}{marker_part}") |
| 151 | + continue |
| 152 | + |
| 153 | + lines.append(f"{name}=={version}{marker_part} \\") |
| 154 | + for i, h in enumerate(hashes): |
| 155 | + suffix = " \\" if i < len(hashes) - 1 else "" |
| 156 | + lines.append(f" --hash={h}{suffix}") |
| 157 | + |
| 158 | + out_path = REPO_ROOT / out_file |
| 159 | + out_path.write_text("\n".join(lines) + "\n") |
| 160 | + log(f" wrote {out_file} ({', '.join(ARCHES)})") |
| 161 | + |
| 162 | + |
| 163 | +def main() -> None: |
| 164 | + """Generate requirements.txt and requirements-build.txt with multi-arch AIPCC hashes.""" |
| 165 | + log("Compiling requirements ...") |
| 166 | + resolved = resolve_packages( |
| 167 | + in_files=["pyproject.toml"], |
| 168 | + extra_args=["--no-emit-package", "kfp-components"], |
| 169 | + ) |
| 170 | + log(f" resolved {len(resolved)} packages, fetching multi-arch hashes ...") |
| 171 | + write_lockfile("requirements.txt", resolved) |
| 172 | + |
| 173 | + log("Compiling requirements-build ...") |
| 174 | + build_deps = _parse_build_requires() |
| 175 | + log(f" build-system requires: {build_deps}") |
| 176 | + resolved = resolve_packages(packages=build_deps) |
| 177 | + log(f" resolved {len(resolved)} packages, fetching multi-arch hashes ...") |
| 178 | + write_lockfile("requirements-build.txt", resolved) |
| 179 | + |
| 180 | + log("Done.") |
| 181 | + |
| 182 | + |
| 183 | +if __name__ == "__main__": |
| 184 | + main() |
0 commit comments