diff --git a/benches/sweep.py b/benches/sweep.py index a226a62..af3c12c 100755 --- a/benches/sweep.py +++ b/benches/sweep.py @@ -50,6 +50,11 @@ SWEEP_GRID = [ (128, 1_200_000), ] +# Number of independent cargo bench processes per measurement point. +# Each process is a fully isolated run (fresh warmup, cold caches, new PID), +# so the final median is a median of independent samples — robust to OS noise. +BENCH_SETS = 5 + # Regression threshold: warn if median is more than this % worse than baseline. REGRESSION_THRESHOLD_PCT = 10 @@ -103,9 +108,12 @@ def parse_output(text: str) -> dict[str, dict[str, dict]]: # Running # --------------------------------------------------------------------------- -def run_benches(env_extra: dict[str, str] | None = None) -> dict[str, dict[str, dict]]: - """Run all BENCHES and return merged parsed results.""" +def run_benches_once(env_extra: dict[str, str] | None = None) -> dict[str, dict[str, dict]]: + """Run all BENCHES once and return merged parsed results.""" env = os.environ.copy() + # Each process does exactly one set of ITERS samples — no within-process + # accumulation; the caller handles multi-set aggregation. + env["SMARM_BENCH_SETS"] = "1" if env_extra: env.update(env_extra) @@ -129,6 +137,45 @@ def run_benches(env_extra: dict[str, str] | None = None) -> dict[str, dict[str, return all_results +def run_benches(env_extra: dict[str, str] | None = None, sets: int = BENCH_SETS) -> dict[str, dict[str, dict]]: + """Run BENCH_SETS independent processes and return median-of-medians per label. + + Each set is a separate cargo bench invocation with its own warmup and OS + context, so samples are statistically independent. The final median and + min/max are computed over the per-set medians. + """ + # Accumulate per-set medians: {bench: {label: [median_set1, median_set2, ...]}} + accumulated: dict[str, dict[str, list[int]]] = {} + last_result: dict[str, dict[str, int]] = {} + + for i in range(sets): + print(f" set {i + 1}/{sets}…", flush=True) + set_results = run_benches_once(env_extra) + for bench, labels in set_results.items(): + accumulated.setdefault(bench, {}) + last_result.setdefault(bench, {}) + for label, data in labels.items(): + accumulated[bench].setdefault(label, []) + accumulated[bench][label].append(data["median"]) + last_result[bench][label] = data["result"] + + # Collapse to final stats. + final: dict[str, dict[str, dict]] = {} + for bench, labels in accumulated.items(): + final[bench] = {} + for label, medians in labels.items(): + medians.sort() + mid = medians[len(medians) // 2] + final[bench][label] = { + "result": last_result[bench][label], + "median": mid, + "min": medians[0], + "max": medians[-1], + } + + return final + + # --------------------------------------------------------------------------- # Baseline JSON # --------------------------------------------------------------------------- @@ -252,7 +299,7 @@ def cmd_run(args) -> None: ["cargo", "build", "--release", "--benches"], cwd=REPO, check=True, capture_output=True, ) - print("Running benches…") + print(f"Running benches ({BENCH_SETS} independent sets)…") results = run_benches() print_results(results, "Results (default knobs)") if args.save_baseline: @@ -266,7 +313,7 @@ def cmd_regress(args) -> None: ["cargo", "build", "--release", "--benches"], cwd=REPO, check=True, capture_output=True, ) - print("Running benches…") + print(f"Running benches ({BENCH_SETS} independent sets)…") current = run_benches() print_results(current, "Current results") print(f"\nRegression check (threshold: >{REGRESSION_THRESHOLD_PCT}% slower than baseline)") @@ -288,7 +335,7 @@ def cmd_sweep(args) -> None: for interval, cycles in SWEEP_GRID: tag = f"alloc_interval={interval}, timeslice_cycles={cycles}" - print(f" Running: {tag} …", flush=True) + print(f" Running: {tag} ({BENCH_SETS} sets)…", flush=True) env_extra = { "SMARM_ALLOC_INTERVAL": str(interval), "SMARM_TIMESLICE_CYCLES": str(cycles),