bench: port urus memory store to gen_server; add spinning A/B harness
- store.rs: MemStore now implements GenServer (one inbox, one hop/request),
replacing the AnyReq + two-forwarder fan-in that cost two extra park/unpark
hops per request and contaminated the spin signal. StoreHandle is now an
enum {Memory(ServerRef<MemStore>) | Sqlite{reads,writes}} with one call().
Response status/body semantics unchanged (S3 stays comparable).
- SQLite store left on raw actors (single-writer + reader pool); not ported
(single-inbox would serialise S4 reads) and not part of the spin A/B. Kept
behind the handle so --store=sqlite still builds.
- handlers.rs: 6 handlers collapse to state.store().call(StoreReq::..); the
per-handler channel+send+recv boilerplate is gone. main.rs untouched.
- spin_ab_urus.sh + analyze_spin_ab_urus.py: A/B the spin policy on urus by
checking out smarm b0c9685 (pre-spin) vs HEAD, S2/S3 on urus-mem, server
cores 1/2/4/8 (2/4 gate-active, 1/8 inert), printing RPS speedup + latency
deltas. Verified: builds release; mem+sqlite smoke pass; scripts dry-run +
analyzer fixture-checked. Real sweep needs the multi-core box.
This commit is contained in:
Executable
+142
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pair before/after urus-bench results and print an A/B comparison.
|
||||
|
||||
Layout produced by spin_ab_urus.sh:
|
||||
|
||||
<outdir>/before/<server>-<scenario>-c<cores>/result.json
|
||||
<outdir>/after/<server>-<scenario>-c<cores>/result.json
|
||||
|
||||
We pair by the (server, scenario, cores) key and report sustained RPS
|
||||
(after/before speedup) and p50/p99/p99.9 latency deltas. The spin gate is
|
||||
active only at 2-4 schedulers, so rows at cores 2 and 4 are where an effect is
|
||||
expected; cores 1 and 8 are controls (gate inert) and should stay flat.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
GATED = {2, 4} # core counts where the RFC 004 spin policy is active
|
||||
|
||||
|
||||
def parse_latency_ms(s):
|
||||
"""wrk2 prints '0.50ms' / '1.83ms' / '24.50ms' / '1.20s' / '850.00us'."""
|
||||
if s is None:
|
||||
return None
|
||||
s = str(s).strip()
|
||||
m = re.match(r"^([0-9.]+)\s*(us|ms|s)?$", s)
|
||||
if not m:
|
||||
return None
|
||||
val = float(m.group(1))
|
||||
unit = m.group(2) or "ms"
|
||||
return {"us": val / 1000.0, "ms": val, "s": val * 1000.0}[unit]
|
||||
|
||||
|
||||
def load(outdir, sub):
|
||||
"""Return {(server, scenario, cores): result_dict} for one side."""
|
||||
base = os.path.join(outdir, sub)
|
||||
out = {}
|
||||
if not os.path.isdir(base):
|
||||
return out
|
||||
for name in os.listdir(base):
|
||||
rj = os.path.join(base, name, "result.json")
|
||||
if not os.path.isfile(rj):
|
||||
continue
|
||||
m = re.match(r"^(.*)-(s\d)-c(\d+)$", name)
|
||||
if not m:
|
||||
continue
|
||||
server, scenario, cores = m.group(1), m.group(2), int(m.group(3))
|
||||
try:
|
||||
with open(rj) as f:
|
||||
out[(server, scenario, cores)] = json.load(f)
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
print(f"warn: could not read {rj}: {e}", file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def fnum(x):
|
||||
try:
|
||||
return float(x)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("outdir")
|
||||
ap.add_argument("--before", default="before")
|
||||
ap.add_argument("--after", default="after")
|
||||
args = ap.parse_args()
|
||||
|
||||
before = load(args.outdir, args.before)
|
||||
after = load(args.outdir, args.after)
|
||||
|
||||
keys = sorted(set(before) | set(after))
|
||||
if not keys:
|
||||
print("no results found under", args.outdir, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# --- RPS table ---------------------------------------------------------
|
||||
hdr = f"{'server':<10} {'scen':<4} {'cores':>5} {'before rps':>12} {'after rps':>12} {'speedup':>8} gate"
|
||||
print("\n== Sustained RPS (higher is better) ==")
|
||||
print(hdr)
|
||||
print("-" * len(hdr))
|
||||
for k in keys:
|
||||
srv, scn, cores = k
|
||||
b = fnum(before.get(k, {}).get("sustained_rps"))
|
||||
a = fnum(after.get(k, {}).get("sustained_rps"))
|
||||
spd = f"{a / b:.2f}x" if (a and b) else "—"
|
||||
gate = "ACTIVE" if cores in GATED else "inert"
|
||||
bs = f"{b:,.0f}" if b is not None else "—"
|
||||
as_ = f"{a:,.0f}" if a is not None else "—"
|
||||
print(f"{srv:<10} {scn:<4} {cores:>5} {bs:>12} {as_:>12} {spd:>8} {gate}")
|
||||
|
||||
# --- latency tables ----------------------------------------------------
|
||||
for pct in ("p50", "p99", "p999"):
|
||||
print(f"\n== {pct} latency, ms (lower is better) ==")
|
||||
h = f"{'server':<10} {'scen':<4} {'cores':>5} {'before':>10} {'after':>10} {'delta':>9} gate"
|
||||
print(h)
|
||||
print("-" * len(h))
|
||||
for k in keys:
|
||||
srv, scn, cores = k
|
||||
b = parse_latency_ms(before.get(k, {}).get("latency", {}).get(pct))
|
||||
a = parse_latency_ms(after.get(k, {}).get("latency", {}).get(pct))
|
||||
if a is not None and b is not None:
|
||||
delta = f"{(a - b):+.2f}"
|
||||
else:
|
||||
delta = "—"
|
||||
gate = "ACTIVE" if cores in GATED else "inert"
|
||||
bs = f"{b:.2f}" if b is not None else "—"
|
||||
as_ = f"{a:.2f}" if a is not None else "—"
|
||||
print(f"{srv:<10} {scn:<4} {cores:>5} {bs:>10} {as_:>10} {delta:>9} {gate}")
|
||||
|
||||
# --- error sanity (non-2xx should be ~0 on both sides) -----------------
|
||||
noisy = []
|
||||
for k in keys:
|
||||
for side, d in (("before", before), ("after", after)):
|
||||
n = d.get(k, {}).get("non_2xx", 0)
|
||||
if isinstance(n, (int, float)) and n > 0:
|
||||
noisy.append((side, k, n))
|
||||
if noisy:
|
||||
print("\n== WARNING: non-2xx responses (results may be unreliable) ==")
|
||||
for side, (srv, scn, cores), n in noisy:
|
||||
print(f" {side} {srv} {scn} c{cores}: {n} non-2xx")
|
||||
|
||||
print(
|
||||
"\nReading it: spin is 'worth it' if s3 (and/or s2) RPS rises and/or "
|
||||
"tail latency drops at cores 2/4 (gate ACTIVE), with cores 1/8 (inert) "
|
||||
"staying flat. Flat-or-worse at 2/4 = not worth it on real traffic."
|
||||
)
|
||||
if "axum-mem" in {k[0] for k in keys}:
|
||||
print(
|
||||
"axum-mem is the control: it doesn't depend on smarm, so its "
|
||||
"before/after should be ~identical. Any drift there means the run "
|
||||
"environment moved, not the patch."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user