Resolve BENCH_DIR at call time in the benchmark runner's main() (#2563)

discover_benchmarks() goes out of its way to avoid def-time binding, and
says so:

    # Resolve the default inside the call so tests (and embedders) can
    # monkeypatch ``BENCH_DIR`` at the module level - Python binds default
    # args at def-time, so a literal default would ignore later patches.
    if bench_dir is None:
        bench_dir = BENCH_DIR

main() then reintroduces exactly that binding:

    def main(
        *,
        bench_dir: Path = BENCH_DIR,
        default_output: Path = DEFAULT_OUTPUT,
        ...
        registry = discover_benchmarks(bench_dir=bench_dir, ...)

Because main() always passes a non-None bench_dir down, the sentinel branch
in discover_benchmarks() can never be taken on this path, and patching
runner.main.BENCH_DIR - the documented mechanism - has no effect on main().
Same for DEFAULT_OUTPUT.

run_pyperf.py calls main() with no arguments, so this is the production
path. The existing tests patch BENCH_DIR and call discover_benchmarks()
directly, which is why the gap is invisible today.

Apply the same sentinel to both parameters. Explicit arguments keep working
unchanged, so the embedder API is unaffected.

Adds test_main_honors_a_monkeypatched_bench_dir, which patches BENCH_DIR to a
tmp dir holding one bench_*.py and drives main() with --list. It fails before
this change (main() lists the repo's real benchmarks instead).

Author: LeSingh1

benchmarks/cuda_bindings/runner/main.py

@@ -232,11 +232,18 @@ def parse_args(argv: list[str], default_output: Path = DEFAULT_OUTPUT) -> tuple[
 
 def main(
     *,
-    bench_dir: Path = BENCH_DIR,
-    default_output: Path = DEFAULT_OUTPUT,
+    bench_dir: Path | None = None,
+    default_output: Path | None = None,
     module_name_prefix: str = DEFAULT_MODULE_NAME_PREFIX,
     bench_filter_env_var: str = DEFAULT_BENCH_FILTER_ENV_VAR,
 ) -> None:
+    # Resolve the defaults inside the call, for the same reason
+    # discover_benchmarks() does: a literal default would be bound at def-time
+    # and would ignore a later monkeypatch of the module-level constant.
+    if bench_dir is None:
+        bench_dir = BENCH_DIR
+    if default_output is None:
+        default_output = DEFAULT_OUTPUT
     parsed, remaining_argv = parse_args(sys.argv[1:], default_output=default_output)
 
     registry = discover_benchmarks(bench_dir=bench_dir, module_name_prefix=module_name_prefix)

benchmarks/cuda_bindings/tests/test_runner.py

@@ -164,3 +164,24 @@ def test_bench_launch_initializes_on_first_use(monkeypatch):
 
     assert len(compile_calls) == 1
     assert len(launch_calls) == 2
+
+
+def test_main_honors_a_monkeypatched_bench_dir(monkeypatch, tmp_path, capsys):
+    """main() must resolve BENCH_DIR at call time, like discover_benchmarks() does.
+
+    A literal default would be bound at def-time and would silently ignore a
+    later patch of the module-level constant.
+    """
+    runner_main = load_runner_main(monkeypatch)
+
+    (tmp_path / "bench_patched.py").write_text(
+        "def bench_only_here(loops: int) -> float:\n    return loops + 0.5\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(runner_main, "BENCH_DIR", tmp_path)
+    runner_main._MODULE_CACHE.clear()
+    monkeypatch.setattr(sys, "argv", ["run_pyperf.py", "--list"])
+
+    runner_main.main()
+
+    assert capsys.readouterr().out.split() == ["patched.only_here"]