benches: add SMARM_BENCH_SETS (default 5) for stable medians

Each run_n call now collects ITERS × SETS raw samples across all sets
(one warmup before the first set, discarded), then takes a single global
median. Previously only ITERS=15 samples were taken per bench invocation,
which produced noisy results in sweep.py regress.

SMARM_BENCH_SETS is read from the environment; the default of 5 gives
75 samples per bench row (5×15), matching the stability of the multi-run
bash approach without requiring multiple cargo bench invocations.

Also moves scripts/bench_rq.sh into benches/ alongside sweep.py.
The cd "$(dirname "$0")/.." path logic is unchanged.
This commit is contained in:
Claude
2026-06-11 20:50:29 +00:00
parent 37d931968e
commit f9f60a43d1
4 changed files with 56 additions and 21 deletions
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# Run-queue shootout driver (ROADMAP_v0.5 phase 4; RFC 005 slot dimension
# added for the v0.9 slot shootout).
#
# Rebuilds the runtime bench once per rq-* feature and runs the raw-structure
# microbench once (it covers all structures in a single binary). The RFC 005
# wake slot is a runtime Config knob, NOT a feature — each rq_runtime binary
# sweeps slot off/on internally (SMARM_BENCH_SLOT, default "0 1"). Results
# land in bench_results/ as full logs; the RQCSV lines are aggregated into
# bench_results/summary.csv and the RQSLOT counter lines (slot hits /
# displacements, slot-on configs only) into bench_results/slot_counters.csv.
#
# Tune the sweep for the box, e.g. on the 20-core machine:
# SMARM_BENCH_THREADS="1 2 4 8 16 20" ./scripts/bench_rq.sh
# Slot-only re-run against the frozen rq-mutex substrate:
# SMARM_BENCH_SLOT="0 1" cargo bench --bench rq_runtime
set -euo pipefail
cd "$(dirname "$0")/.."
OUT=bench_results
mkdir -p "$OUT"
: "${SMARM_BENCH_THREADS:=1 2 4}"
: "${SMARM_BENCH_SLOT:=0 1}"
export SMARM_BENCH_THREADS SMARM_BENCH_SLOT
echo "== raw structures (one binary, all variants) =="
cargo bench --bench rq_micro 2>&1 | tee "$OUT/micro.txt"
for v in rq-mutex rq-mpmc rq-striped; do
echo "== runtime benches: $v (slot sweep: $SMARM_BENCH_SLOT) =="
cargo bench --bench rq_runtime --no-default-features --features "$v" \
2>&1 | tee "$OUT/runtime-$v.txt"
done
# runtime rows: kind,variant,slot,bench,threads,work,median_us,ops_per_s
# micro rows: kind,structure,threads,p:c,items,median_us,items_per_s (one
# column narrower, as before — split on kind when plotting)
echo "kind,a,b,c,d,e,median_us,ops_per_s" > "$OUT/summary.csv"
grep -h '^RQCSV,' "$OUT"/*.txt | sed 's/^RQCSV,//' >> "$OUT/summary.csv"
echo "variant,bench,threads,slot_hits,slot_displacements" > "$OUT/slot_counters.csv"
grep -h '^RQSLOT,' "$OUT"/*.txt | sed 's/^RQSLOT,//' >> "$OUT/slot_counters.csv" || true
echo
echo "Summary: $OUT/summary.csv ($(($(wc -l < "$OUT/summary.csv") - 1)) rows)"
echo "Slot counters: $OUT/slot_counters.csv ($(($(wc -l < "$OUT/slot_counters.csv") - 1)) rows)"
+18 -7
View File
@@ -29,6 +29,13 @@ fn available_threads() -> usize {
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)
}
fn env_sets() -> u32 {
std::env::var("SMARM_BENCH_SETS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5)
}
fn print_header(title: &str) {
println!("\n{}", "=".repeat(80));
println!(" {title}");
@@ -41,14 +48,17 @@ fn print_header(title: &str) {
}
fn run_n<F: FnMut() -> (u64, u128)>(name: &str, n: u32, mut f: F) {
let mut times = Vec::new();
let sets = env_sets();
let mut times = Vec::with_capacity((n * sets) as usize);
let mut last = 0u64;
// One warmup iteration, discarded.
// One warmup before all sets, discarded.
let _ = f();
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
for _ in 0..sets {
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
}
}
times.sort_unstable();
let median = times[times.len() / 2];
@@ -406,7 +416,8 @@ fn main() {
let n = available_threads();
println!("smarm general benchmarks");
println!("available parallelism: {n} threads");
println!("ITERS={ITERS} (+1 warmup, discarded)");
let sets = env_sets();
println!("ITERS={ITERS}×{sets} sets = {} samples (+1 warmup, discarded)", ITERS * sets);
println!(
"CHAIN_DEPTH={CHAIN_DEPTH}, YIELD_TASKS={YIELD_TASKS}×{YIELD_ROUNDS}, \
PRIME_N={PRIME_N}/{PRIME_WORKERS} workers, PP_ROUNDS={PP_ROUNDS}"
+19 -7
View File
@@ -40,6 +40,13 @@ fn available_threads() -> usize {
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)
}
fn env_sets() -> u32 {
std::env::var("SMARM_BENCH_SETS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5)
}
fn print_header(title: &str) {
println!("\n{}", "=".repeat(80));
println!(" {title}");
@@ -52,13 +59,17 @@ fn print_header(title: &str) {
}
fn run_n<F: FnMut() -> (u64, u128)>(name: &str, n: u32, mut f: F) {
let mut times = Vec::new();
let sets = env_sets();
let mut times = Vec::with_capacity((n * sets) as usize);
let mut last = 0u64;
let _ = f(); // warmup
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
// One warmup before all sets, discarded.
let _ = f();
for _ in 0..sets {
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
}
}
times.sort_unstable();
let median = times[times.len() / 2];
@@ -385,7 +396,8 @@ fn main() {
let n = available_threads();
println!("smarm smarm-favored benchmarks");
println!("available parallelism: {n} threads");
println!("ITERS={ITERS} (+1 warmup, discarded)");
let sets = env_sets();
println!("ITERS={ITERS}×{sets} sets = {} samples (+1 warmup, discarded)", ITERS * sets);
println!(
"RECURSE_DEPTH={RECURSE_DEPTH}, HOT_YIELDS={HOT_YIELDS}×2, \
UNCONT_MSGS={UNCONT_MSGS}, PANIC_TASKS={PANIC_TASKS}"
+19 -7
View File
@@ -39,6 +39,13 @@ fn available_threads() -> usize {
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)
}
fn env_sets() -> u32 {
std::env::var("SMARM_BENCH_SETS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(5)
}
fn print_header(title: &str) {
println!("\n{}", "=".repeat(80));
println!(" {title}");
@@ -51,13 +58,17 @@ fn print_header(title: &str) {
}
fn run_n<F: FnMut() -> (u64, u128)>(name: &str, n: u32, mut f: F) {
let mut times = Vec::new();
let sets = env_sets();
let mut times = Vec::with_capacity((n * sets) as usize);
let mut last = 0u64;
let _ = f(); // warmup
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
// One warmup before all sets, discarded.
let _ = f();
for _ in 0..sets {
for _ in 0..n {
let (v, t) = f();
times.push(t);
last = v;
}
}
times.sort_unstable();
let median = times[times.len() / 2];
@@ -434,7 +445,8 @@ fn main() {
let n = available_threads();
println!("smarm tokio-favored benchmarks");
println!("available parallelism: {n} threads");
println!("ITERS={ITERS} (+1 warmup, discarded)");
let sets = env_sets();
println!("ITERS={ITERS}×{sets} sets = {} samples (+1 warmup, discarded)", ITERS * sets);
println!(
"STORM_BACKGROUND={STORM_BACKGROUND}, STORM_SPAWN={STORM_SPAWN}, \
MPSC={MPSC_PRODUCERS}×{MPSC_PER_PRODUCER}, \