From db76d22a1600295a7ae057343cd857c24cdee35b Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Tue, 14 Jan 2025 07:31:06 -0800 Subject: [PATCH 1/5] add stdev to perf_compare --- misc/perf_compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index ef9976b8e2eb..310ff0bd4340 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -171,13 +171,14 @@ def main() -> None: first = -1.0 for commit in commits: tt = statistics.mean(results[commit]) + s = statistics.stdev(results[commit]) if first < 0: delta = "0.0%" first = tt else: d = (tt / first) - 1 delta = f"{d:+.1%}" - print(f"{commit:<25} {tt:.3f}s ({delta})") + print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s}") shutil.rmtree(self_check_dir) for target_dir in target_dirs: From 3fad40f443b57844e0bdc12726ab51fdce56117b Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 15 Jan 2025 01:01:40 -0800 Subject: [PATCH 2/5] add -r, --dont-setup, epilog, and total time --- misc/perf_compare.py | 77 ++++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 310ff0bd4340..952ddd789565 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -44,13 +44,15 @@ def build_mypy(target_dir: str) -> None: subprocess.run(cmd, env=env, check=True, cwd=target_dir) -def clone(target_dir: str, commit: str | None) -> None: - heading(f"Cloning mypy to {target_dir}") - repo_dir = os.getcwd() +def clone(target_dir: str, commit: str | None, repo_source: str | None = None) -> None: + source_name = repo_source or "mypy" + heading(f"Cloning {source_name} to {target_dir}") + if repo_source is None: + repo_source = os.getcwd() if os.path.isdir(target_dir): print(f"{target_dir} exists: deleting") shutil.rmtree(target_dir) - subprocess.run(["git", "clone", repo_dir, target_dir], check=True) + subprocess.run(["git", "clone", repo_source, target_dir], check=True) if commit: subprocess.run(["git", "checkout", commit], check=True, cwd=target_dir) @@ -64,7 +66,7 @@ def edit_python_file(fnam: str) -> None: def run_benchmark( - compiled_dir: str, check_dir: str, *, incremental: bool, code: str | None + compiled_dir: str, check_dir: str, *, incremental: bool, code: str, foreign: bool | None ) -> float: cache_dir = os.path.join(compiled_dir, ".mypy_cache") if os.path.isdir(cache_dir) and not incremental: @@ -76,6 +78,8 @@ def run_benchmark( cmd = [sys.executable, "-m", "mypy"] if code: cmd += ["-c", code] + elif foreign: + pass else: cmd += ["--config-file", os.path.join(abschk, "mypy_self_check.ini")] cmd += glob.glob(os.path.join(abschk, "mypy/*.py")) @@ -86,18 +90,28 @@ def run_benchmark( edit_python_file(os.path.join(abschk, "mypy/test/testcheck.py")) t0 = time.time() # Ignore errors, since some commits being measured may generate additional errors. - subprocess.run(cmd, cwd=compiled_dir, env=env) + if foreign: + subprocess.run(cmd, cwd=check_dir, env=env) + else: + subprocess.run(cmd, cwd=compiled_dir, env=env) return time.time() - t0 def main() -> None: - parser = argparse.ArgumentParser() + whole_program_time_0 = time.time() + parser = argparse.ArgumentParser(epilog="Remember: you usually want the first argument to this command to be 'master'.") parser.add_argument( "--incremental", default=False, action="store_true", help="measure incremental run (fully cached)", ) + parser.add_argument( + "--dont-setup", + default=False, + action="store_true", + help="don't make the dirs or compile mypy, just run the performance measurement benchmark", + ) parser.add_argument( "--num-runs", metavar="N", @@ -112,6 +126,14 @@ def main() -> None: type=int, help="set maximum number of parallel builds (default=8)", ) + parser.add_argument( + "-r", + metavar="FOREIGN_REPOSITORY", + default=None, + type=str, + help="measure time to type check the project at FOREIGN_REPOSITORY instead of mypy self-check; " + + "provided value must be the URL or path of a git repo", + ) parser.add_argument( "-c", metavar="CODE", @@ -122,10 +144,12 @@ def main() -> None: parser.add_argument("commit", nargs="+", help="git revision to measure (e.g. branch name)") args = parser.parse_args() incremental: bool = args.incremental + dont_setup: bool = args.dont_setup commits = args.commit num_runs: int = args.num_runs + 1 max_workers: int = args.j code: str | None = args.c + foreign_repo: str | None = args.r if not (os.path.isdir(".git") and os.path.isdir("mypyc")): sys.exit("error: Run this the mypy repo root") @@ -134,20 +158,28 @@ def main() -> None: for i, commit in enumerate(commits): target_dir = f"mypy.{i}.tmpdir" target_dirs.append(target_dir) - clone(target_dir, commit) + if not dont_setup: + clone(target_dir, commit) - self_check_dir = "mypy.self.tmpdir" - clone(self_check_dir, commits[0]) + if foreign_repo: + check_dir = "mypy.foreign.tmpdir" + if not dont_setup: + clone(check_dir, None, foreign_repo) + else: + check_dir = "mypy.self.tmpdir" + if not dont_setup: + clone(check_dir, commits[0]) - heading("Compiling mypy") - print("(This will take a while...)") + if not dont_setup: + heading("Compiling mypy") + print("(This will take a while...)") - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs] - for future in as_completed(futures): - future.result() + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs] + for future in as_completed(futures): + future.result() - print(f"Finished compiling mypy ({len(commits)} builds)") + print(f"Finished compiling mypy ({len(commits)} builds)") heading("Performing measurements") @@ -160,7 +192,7 @@ def main() -> None: items = list(enumerate(commits)) random.shuffle(items) for i, commit in items: - tt = run_benchmark(target_dirs[i], self_check_dir, incremental=incremental, code=code) + tt = run_benchmark(target_dirs[i], check_dir, incremental=incremental, code=code, foreign=bool(foreign_repo)) # Don't record the first warm-up run if n > 0: print(f"{commit}: t={tt:.3f}s") @@ -171,16 +203,19 @@ def main() -> None: first = -1.0 for commit in commits: tt = statistics.mean(results[commit]) - s = statistics.stdev(results[commit]) + #pstdev (instead of stdev) is used here primarily to accommodate the case where num_runs=1 + s = statistics.pstdev(results[commit]) if len(results[commit]) > 1 else 0 if first < 0: delta = "0.0%" first = tt else: d = (tt / first) - 1 delta = f"{d:+.1%}" - print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s}") + print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s:.3f}s ") + + print(f"Total time taken by the benchmarking program (including any setup): {time.time() - whole_program_time_0:.2f}s") - shutil.rmtree(self_check_dir) + shutil.rmtree(check_dir) for target_dir in target_dirs: shutil.rmtree(target_dir) From f56c230e77efe39542d53ef873a6fc35aac38fe9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 15 Jan 2025 01:24:58 -0800 Subject: [PATCH 3/5] human format the time elapsed better --- misc/perf_compare.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 952ddd789565..888855d8040f 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -213,7 +213,11 @@ def main() -> None: delta = f"{d:+.1%}" print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s:.3f}s ") - print(f"Total time taken by the benchmarking program (including any setup): {time.time() - whole_program_time_0:.2f}s") + t = int( time.time() - whole_program_time_0 ) + total_time_taken_formatted = ", ".join( + f"{v} {n if v==1 else n+'s'}" for v, n in ((t//3600, "hour"), (t//60%60, "minute"), (t%60, "second")) if v + ) + print(f"Total time taken by the whole benchmarking program (including any setup):", total_time_taken_formatted) shutil.rmtree(check_dir) for target_dir in target_dirs: From 93526c2c1759bcb0e6e294c4c12b4960a4e38b3e Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Wed, 15 Jan 2025 01:27:10 -0800 Subject: [PATCH 4/5] correct three editing errors --- misc/perf_compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index 888855d8040f..b18bc9e08123 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -66,7 +66,7 @@ def edit_python_file(fnam: str) -> None: def run_benchmark( - compiled_dir: str, check_dir: str, *, incremental: bool, code: str, foreign: bool | None + compiled_dir: str, check_dir: str, *, incremental: bool, code: str | None, foreign: bool | None ) -> float: cache_dir = os.path.join(compiled_dir, ".mypy_cache") if os.path.isdir(cache_dir) and not incremental: @@ -213,11 +213,11 @@ def main() -> None: delta = f"{d:+.1%}" print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s:.3f}s ") - t = int( time.time() - whole_program_time_0 ) + t = int( time.time() - whole_program_time_0 ) total_time_taken_formatted = ", ".join( f"{v} {n if v==1 else n+'s'}" for v, n in ((t//3600, "hour"), (t//60%60, "minute"), (t%60, "second")) if v ) - print(f"Total time taken by the whole benchmarking program (including any setup):", total_time_taken_formatted) + print("Total time taken by the whole benchmarking program (including any setup):", total_time_taken_formatted) shutil.rmtree(check_dir) for target_dir in target_dirs: From 4bbd45e510ae0d6e1b22e02f4656f89abc15b9ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:37:14 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- misc/perf_compare.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/misc/perf_compare.py b/misc/perf_compare.py index b18bc9e08123..2716ecbfbabf 100644 --- a/misc/perf_compare.py +++ b/misc/perf_compare.py @@ -99,7 +99,9 @@ def run_benchmark( def main() -> None: whole_program_time_0 = time.time() - parser = argparse.ArgumentParser(epilog="Remember: you usually want the first argument to this command to be 'master'.") + parser = argparse.ArgumentParser( + epilog="Remember: you usually want the first argument to this command to be 'master'." + ) parser.add_argument( "--incremental", default=False, @@ -131,8 +133,8 @@ def main() -> None: metavar="FOREIGN_REPOSITORY", default=None, type=str, - help="measure time to type check the project at FOREIGN_REPOSITORY instead of mypy self-check; " + - "provided value must be the URL or path of a git repo", + help="measure time to type check the project at FOREIGN_REPOSITORY instead of mypy self-check; " + + "provided value must be the URL or path of a git repo", ) parser.add_argument( "-c", @@ -192,7 +194,13 @@ def main() -> None: items = list(enumerate(commits)) random.shuffle(items) for i, commit in items: - tt = run_benchmark(target_dirs[i], check_dir, incremental=incremental, code=code, foreign=bool(foreign_repo)) + tt = run_benchmark( + target_dirs[i], + check_dir, + incremental=incremental, + code=code, + foreign=bool(foreign_repo), + ) # Don't record the first warm-up run if n > 0: print(f"{commit}: t={tt:.3f}s") @@ -203,7 +211,7 @@ def main() -> None: first = -1.0 for commit in commits: tt = statistics.mean(results[commit]) - #pstdev (instead of stdev) is used here primarily to accommodate the case where num_runs=1 + # pstdev (instead of stdev) is used here primarily to accommodate the case where num_runs=1 s = statistics.pstdev(results[commit]) if len(results[commit]) > 1 else 0 if first < 0: delta = "0.0%" @@ -213,11 +221,16 @@ def main() -> None: delta = f"{d:+.1%}" print(f"{commit:<25} {tt:.3f}s ({delta}) | stdev {s:.3f}s ") - t = int( time.time() - whole_program_time_0 ) + t = int(time.time() - whole_program_time_0) total_time_taken_formatted = ", ".join( - f"{v} {n if v==1 else n+'s'}" for v, n in ((t//3600, "hour"), (t//60%60, "minute"), (t%60, "second")) if v + f"{v} {n if v==1 else n+'s'}" + for v, n in ((t // 3600, "hour"), (t // 60 % 60, "minute"), (t % 60, "second")) + if v + ) + print( + "Total time taken by the whole benchmarking program (including any setup):", + total_time_taken_formatted, ) - print("Total time taken by the whole benchmarking program (including any setup):", total_time_taken_formatted) shutil.rmtree(check_dir) for target_dir in target_dirs: