Refactor so as much work as possible is done in Python

Author: mdboom

Diff for: bench_runner/__main__.py

@@ -14,12 +14,12 @@
         "Get the merge base of the selected commit, and determine if it should run"
     ),
     "install": "Install the workflow files into a results repository",
+    "notify": "Send a notification about the completion of the workflow",
     "profiling_plot": "Generate the profiling plots from raw data",
     "purge": "Purge old results from a results repository",
     "remove_benchmark": "Remove specific benchmarks from the data set",
     "run_benchmarks": "Run benchmarks (in timing, pyperf or perf modes)",
-    "should_run": "Determine whether we need to rerun results for the current commit",
-    "notify": "Send a notification about the completion of the workflow",
+    "workflow": "Run the full compile/benchmark workflow",
 }
 
 if __name__ == "__main__":

Diff for: bench_runner/benchmark_definitions.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+
+import dataclasses
+import hashlib
+from pathlib import Path
+
+
+from . import git
+
+
+@dataclasses.dataclass
+class BenchmarkRepo:
+    hash: str
+    url: str
+    dirname: str
+
+
+BENCHMARK_REPOS = [
+    BenchmarkRepo(
+        "56d12a8fd7cc1432835965d374929bfa7f6f7a07",
+        "https://github.com/mdboom/pyperformance.git",
+        "pyperformance",
+    ),
+    BenchmarkRepo(
+        "265655e7f03ace13ec1e00e1ba299179e69f8a00",
+        "https://github.com/pyston/python-macrobenchmarks.git",
+        "pyston-benchmarks",
+    ),
+]
+
+
+def get_benchmark_hash() -> str:
+    hash = hashlib.sha256()
+    for repo in BENCHMARK_REPOS:
+        if Path(repo.dirname).is_dir():
+            current_hash = git.get_git_hash(Path(repo.dirname))
+        else:
+            current_hash = repo.hash
+        hash.update(current_hash.encode("ascii")[:7])
+    return hash.hexdigest()[:6]

Diff for: bench_runner/git.py

@@ -2,9 +2,12 @@
 from __future__ import annotations
 
 
+import contextlib
 import datetime
 from pathlib import Path
+import shutil
 import subprocess
+import re
 
 
 import rich
@@ -128,3 +131,40 @@ def get_commits_between(dirname: PathLike, ref1: str, ref2: str) -> list[str]:
 def bisect_commits(dirname: PathLike, ref1: str, ref2: str) -> str:
     commits = get_commits_between(dirname, ref1, ref2)
     return commits[len(commits) // 2]
+
+
+def clone(
+    dirname: PathLike,
+    url: str,
+    *,
+    branch: str | None = None,
+    depth: int = 1,
+) -> None:
+    is_hash = re.match(r"^[0-9a-f]{40}$", branch) if branch else False
+
+    dirname = Path(dirname)
+    if dirname.is_dir():
+        if is_hash and (dirname / ".git").is_dir() and get_git_hash(dirname) == branch:
+            # This is a git repo, and the hash matches
+            return
+        shutil.rmtree(dirname)
+
+    # Fetching a hash and fetching a branch require different approaches
+
+    if is_hash:
+        assert branch is not None
+        dirname.mkdir()
+        with contextlib.chdir(dirname):
+            subprocess.check_call(["git", "init"])
+            subprocess.check_call(["git", "remote", "add", "origin", url])
+            subprocess.check_call(
+                ["git", "fetch", "--depth", str(depth), "origin", branch]
+            )
+            subprocess.check_call(["git", "checkout", branch])
+    else:
+        args = ["git", "clone", url, str(dirname)]
+        if branch is not None:
+            args += ["--branch", branch]
+        if depth is not None:
+            args += ["--depth", str(depth)]
+        subprocess.check_call(args)

Diff for: bench_runner/scripts/get_merge_base.py

@@ -6,10 +6,10 @@
 import rich_argparse
 
 
+from bench_runner import benchmark_definitions
 from bench_runner import flags as mflags
 from bench_runner import git
 from bench_runner.result import has_result
-from bench_runner import util
 from bench_runner.util import PathLike
 
 
@@ -55,7 +55,7 @@ def _main(
                     machine,
                     pystats,
                     flags,
-                    util.get_benchmark_hash(),
+                    benchmark_definitions.get_benchmark_hash(),
                     progress=False,
                 )
                 is None

Diff for: bench_runner/scripts/install.py

@@ -241,13 +241,11 @@ def generate_generic(dst: Any) -> Any:
 def _main(check: bool) -> None:
     WORKFLOW_PATH.mkdir(parents=True, exist_ok=True)
 
-    env = load_yaml(TEMPLATE_PATH / "env.yml")
-
     for path in TEMPLATE_PATH.glob("*"):
         if path.name.endswith(".src.yml") or path.name == "env.yml":
             continue
 
-        if not (ROOT_PATH / path.name).is_file():
+        if not (ROOT_PATH / path.name).is_file() or path.suffix == ".py":
             if check:
                 fail_check(ROOT_PATH / path.name)
             else:
@@ -258,7 +256,6 @@ def _main(check: bool) -> None:
         generator = GENERATORS.get(src_path.name, generate_generic)
         src = load_yaml(src_path)
         dst = generator(src)
-        dst = {"env": env, **dst}
         write_yaml(dst_path, dst, check)
 
 

Diff for: bench_runner/scripts/run_benchmarks.py

@@ -18,6 +18,7 @@
 import rich_argparse
 
 
+from bench_runner import benchmark_definitions
 from bench_runner import flags
 from bench_runner import git
 from bench_runner.result import Result
@@ -265,7 +266,7 @@ def update_metadata(
     merge_base = git.get_git_merge_base(cpython)
     if merge_base is not None:
         metadata["commit_merge_base"] = merge_base
-    metadata["benchmark_hash"] = util.get_benchmark_hash()
+    metadata["benchmark_hash"] = benchmark_definitions.get_benchmark_hash()
     if run_id is not None:
         metadata["github_action_url"] = f"{GITHUB_URL}/actions/runs/{run_id}"
     actor = os.environ.get("GITHUB_ACTOR")