benches: stable medians via independent process sets in sweep.py

Replace single-process run_benches() with a two-layer approach:

- run_benches_once(): one cargo bench invocation with SMARM_BENCH_SETS=1,
  yielding one median per label from ITERS raw samples.
- run_benches(): calls run_benches_once() BENCH_SETS=5 times, accumulates
  the per-set medians as independent samples, and returns the median of
  those 5 values.

Each set is a fully isolated process (fresh PID, cold caches, independent
OS scheduler context, new warmup run), so the 5 samples are statistically
independent. Correlated noise within a single sustained process run — CPU
frequency scaling, sticky OS scheduler decisions, warm branch predictors —
no longer contaminates the final median.

The previous fix (SMARM_BENCH_SETS loop inside run_n) collected more
samples but within one process, giving correlated samples that were still
vulnerable to a noisy measurement window for any one label.

benches/sweep.py

@@ -50,6 +50,11 @@ SWEEP_GRID = [
     (128, 1_200_000),
 ]
 
+# Number of independent cargo bench processes per measurement point.
+# Each process is a fully isolated run (fresh warmup, cold caches, new PID),
+# so the final median is a median of independent samples — robust to OS noise.
+BENCH_SETS = 5
+
 # Regression threshold: warn if median is more than this % worse than baseline.
 REGRESSION_THRESHOLD_PCT = 10
 
@@ -103,9 +108,12 @@ def parse_output(text: str) -> dict[str, dict[str, dict]]:
 # Running
 # ---------------------------------------------------------------------------
 
-def run_benches(env_extra: dict[str, str] | None = None) -> dict[str, dict[str, dict]]:
+def run_benches_once(env_extra: dict[str, str] | None = None) -> dict[str, dict[str, dict]]:
-    """Run all BENCHES and return merged parsed results."""
+    """Run all BENCHES once and return merged parsed results."""
     env = os.environ.copy()
+    # Each process does exactly one set of ITERS samples — no within-process
+    # accumulation; the caller handles multi-set aggregation.
+    env["SMARM_BENCH_SETS"] = "1"
     if env_extra:
         env.update(env_extra)
 
@@ -129,6 +137,45 @@ def run_benches(env_extra: dict[str, str] | None = None) -> dict[str, dict[str,
     return all_results
 
 
+def run_benches(env_extra: dict[str, str] | None = None, sets: int = BENCH_SETS) -> dict[str, dict[str, dict]]:
+    """Run BENCH_SETS independent processes and return median-of-medians per label.
+
+    Each set is a separate cargo bench invocation with its own warmup and OS
+    context, so samples are statistically independent. The final median and
+    min/max are computed over the per-set medians.
+    """
+    # Accumulate per-set medians: {bench: {label: [median_set1, median_set2, ...]}}
+    accumulated: dict[str, dict[str, list[int]]] = {}
+    last_result: dict[str, dict[str, int]] = {}
+
+    for i in range(sets):
+        print(f"    set {i + 1}/{sets}…", flush=True)
+        set_results = run_benches_once(env_extra)
+        for bench, labels in set_results.items():
+            accumulated.setdefault(bench, {})
+            last_result.setdefault(bench, {})
+            for label, data in labels.items():
+                accumulated[bench].setdefault(label, [])
+                accumulated[bench][label].append(data["median"])
+                last_result[bench][label] = data["result"]
+
+    # Collapse to final stats.
+    final: dict[str, dict[str, dict]] = {}
+    for bench, labels in accumulated.items():
+        final[bench] = {}
+        for label, medians in labels.items():
+            medians.sort()
+            mid = medians[len(medians) // 2]
+            final[bench][label] = {
+                "result": last_result[bench][label],
+                "median": mid,
+                "min":    medians[0],
+                "max":    medians[-1],
+            }
+
+    return final
+
+
 # ---------------------------------------------------------------------------
 # Baseline JSON
 # ---------------------------------------------------------------------------
@@ -252,7 +299,7 @@ def cmd_run(args) -> None:
         ["cargo", "build", "--release", "--benches"],
         cwd=REPO, check=True, capture_output=True,
     )
-    print("Running benches…")
+    print(f"Running benches ({BENCH_SETS} independent sets)…")
     results = run_benches()
     print_results(results, "Results (default knobs)")
     if args.save_baseline:
@@ -266,7 +313,7 @@ def cmd_regress(args) -> None:
         ["cargo", "build", "--release", "--benches"],
         cwd=REPO, check=True, capture_output=True,
     )
-    print("Running benches…")
+    print(f"Running benches ({BENCH_SETS} independent sets)…")
     current = run_benches()
     print_results(current, "Current results")
     print(f"\nRegression check (threshold: >{REGRESSION_THRESHOLD_PCT}% slower than baseline)")
@@ -288,7 +335,7 @@ def cmd_sweep(args) -> None:
 
     for interval, cycles in SWEEP_GRID:
         tag = f"alloc_interval={interval}, timeslice_cycles={cycles}"
-        print(f"  Running: {tag} …", flush=True)
+        print(f"  Running: {tag} ({BENCH_SETS} sets)…", flush=True)
         env_extra = {
             "SMARM_ALLOC_INTERVAL":    str(interval),
             "SMARM_TIMESLICE_CYCLES":  str(cycles),