From 8701ccb9847ca2437220dacae03773eef0b20736 Mon Sep 17 00:00:00 2001 From: User Name Date: Fri, 14 Aug 2026 19:17:25 -0400 Subject: [PATCH 01/11] feat(mmbt): support remote Qwen3.8 dual-lane campaigns --- tooling/bench_autopilot.py | 18 +- tooling/bench_dashboard.py | 13 +- tooling/bench_report.py | 77 +++++++- tooling/harness.py | 175 ++++++++++++++---- tooling/qwen3.8-27b-q4-t1-t3-mmbt.json | 51 +++++ tooling/scripts/run_microbench.sh | 46 ++++- tooling/test_bench_autopilot_sampling.py | 35 ++++ tooling/test_bench_dashboard_paths.py | 15 ++ tooling/test_bench_report_config.py | 39 ++++ tooling/test_harness_qwen38_compat.py | 87 +++++++++ .../test_run_microbench_qwen38_transport.py | 68 +++++++ 11 files changed, 562 insertions(+), 62 deletions(-) create mode 100644 tooling/qwen3.8-27b-q4-t1-t3-mmbt.json create mode 100644 tooling/test_bench_dashboard_paths.py create mode 100644 tooling/test_bench_report_config.py create mode 100644 tooling/test_harness_qwen38_compat.py create mode 100644 tooling/test_run_microbench_qwen38_transport.py diff --git a/tooling/bench_autopilot.py b/tooling/bench_autopilot.py index 83135b61..08ada863 100755 --- a/tooling/bench_autopilot.py +++ b/tooling/bench_autopilot.py @@ -827,7 +827,7 @@ def git(*args, check=False): log(f"publish: git step failed (non-fatal): {e}") -def benchmark_environment(cfg, base_env=None): +def benchmark_environment(cfg, base_env=None, arm=None): """Return the child environment carrying a model's pinned sampling point. ``run_microbench.sh`` intentionally keeps historical defaults, so model-card @@ -840,13 +840,20 @@ def benchmark_environment(cfg, base_env=None): "benchmark_temperature": "BENCH_TEMP", "benchmark_top_p": "BENCH_TOP_P", "benchmark_top_k": "BENCH_TOP_K", + "benchmark_min_p": "BENCH_MIN_P", + "benchmark_presence_penalty": "BENCH_PRESENCE_PENALTY", + "benchmark_repeat_penalty": "BENCH_REPEAT_PENALTY", + "benchmark_seed": "BENCH_SEED", "benchmark_max_output_tokens_cap": "BENCH_MAX_OUTPUT_TOKENS_CAP", + "benchmark_sandbox_gpus": "BENCH_SANDBOX_GPUS", + "preserve_thinking": "BENCH_PRESERVE_THINKING", + "reasoning_effort_location": "BENCH_REASONING_EFFORT_LOCATION", "serving_manifest": "BENCH_SERVING_MANIFEST", } for config_key, env_key in sampling_env.items(): - value = cfg.get(config_key) + value = arm.get(config_key, cfg.get(config_key)) if arm else cfg.get(config_key) if value is not None: - run_env[env_key] = str(value) + run_env[env_key] = str(value).lower() if isinstance(value, bool) else str(value) else: run_env.pop(env_key, None) return run_env @@ -866,7 +873,7 @@ def run_arm_with_supervision(cfg, arm, target, started_at): handles = [] try: for lane_index, port in enumerate(ports): - run_env = benchmark_environment(cfg) + run_env = benchmark_environment(cfg, arm=arm) run_env["BENCH_LANE_INDEX"] = str(lane_index) run_env["BENCH_LANE_COUNT"] = str(lane_count) log_path = STATE_DIR / ( @@ -877,7 +884,8 @@ def run_arm_with_supervision(cfg, arm, target, started_at): handles.append(handle) proc = subprocess.Popen( ["bash", str(SCRIPTS / "run_microbench.sh"), cfg["model"], str(port), - label, str(target), "", thinking, str(cfg["max_model_len"])], + label, str(target), str(arm.get("reasoning_effort", "")), + thinking, str(cfg["max_model_len"])], stdout=handle, stderr=subprocess.STDOUT, env=run_env) procs.append((lane_index, port, proc)) diff --git a/tooling/bench_dashboard.py b/tooling/bench_dashboard.py index 13a1f810..8f473dbc 100644 --- a/tooling/bench_dashboard.py +++ b/tooling/bench_dashboard.py @@ -33,14 +33,15 @@ from collections import Counter from pathlib import Path -HOME = Path(os.path.expanduser("~")) -LOGS = HOME / "bench" / "logs" +REPO = Path(__file__).resolve().parent.parent +LOGS = REPO / "logs" STATE = Path("/tmp/bench-autopilot") STATUS = STATE / "status.json" LOG = STATE / "autopilot.log" -# Glob that matches a cell dir for any arm/rep (e.g. p1_bugfix_397b-nothink_v3). -CELL_GLOB = "p*_397b-*_v*" +# Glob that matches a cell dir for any model arm/rep. A dedicated campaign +# checkout keeps this generic scan isolated from unrelated historical logs. +CELL_GLOB = "p[1-3]_*_v*" # N=10 no-think baseline pass counts per task (totals to 82/120). BASELINE_N10_NOTHINK = { @@ -536,6 +537,7 @@ def load_status(): def main(): + global LOGS ap = argparse.ArgumentParser() ap.add_argument("--watch", action="store_true") ap.add_argument("--html", default=None) @@ -550,7 +552,10 @@ def main(): help="per-task cumulative-window pass-rate table (v1-3/5/10/20) per arm") ap.add_argument("--flips", action="store_true", help="per-task flip map: cells differing from the task/arm modal verdict") + ap.add_argument("--logs", default=str(LOGS), + help="per-cell logs directory (default: logs in this checkout)") args = ap.parse_args() + LOGS = Path(os.path.expanduser(args.logs)).resolve() def once(): status = load_status() # None if missing/partial — every renderer handles None diff --git a/tooling/bench_report.py b/tooling/bench_report.py index 726022ab..ab1b1f15 100644 --- a/tooling/bench_report.py +++ b/tooling/bench_report.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """bench_report — standalone MMBT findings-markdown generator. -Reads ~/bench/logs/_