#!/usr/bin/env bash # urus-bench/runner.sh # # Runs one (server, scenario) measurement and saves all artefacts under # results/${run_id}/. The intent is to make a single run trivially # repeatable so the headline table in the bench-spec can be filled in # row by row. # # Usage: # ./runner.sh [run_id] # # server: # urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite # scenario: # s1 | s2 | s3 | s4 # run_id: # optional; defaults to ${date}-${server}-${scenario} # # Requirements on the host: # - taskset (util-linux) # - wrk2 in PATH (https://github.com/giltene/wrk2) # - wrk in PATH (saturation probe) # - pidstat (sysstat) -- optional, CPU stats only # - jq -- pretty-prints the result.json # # CPU pinning (defaults; override with SERVER_CPUS / LOADGEN_CPUS): # server: taskset -c 0-7 # loadgen: taskset -c 8-15 # # Probe / measurement times: 30s probe, 30s warm-up, 60s measured (per the # bench-spec ยง1.5). Override with PROBE_SEC / WARMUP_SEC / MEASURE_SEC. set -euo pipefail # --------------------------------------------------------------------------- # Args / config # --------------------------------------------------------------------------- SERVER="${1:-}" SCENARIO="${2:-}" RUN_ID="${3:-$(date +%Y%m%d-%H%M%S)-${SERVER}-${SCENARIO}}" if [[ -z "$SERVER" || -z "$SCENARIO" ]]; then echo "usage: $0 [run_id]" >&2 echo " server: urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite" >&2 echo " scenario: s1 | s2 | s3 | s4" >&2 exit 2 fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RESULTS_DIR="${REPO_ROOT}/results/${RUN_ID}" mkdir -p "$RESULTS_DIR" SERVER_CPUS="${SERVER_CPUS:-0-7}" LOADGEN_CPUS="${LOADGEN_CPUS:-8-15}" PROBE_SEC="${PROBE_SEC:-30}" WARMUP_SEC="${WARMUP_SEC:-30}" MEASURE_SEC="${MEASURE_SEC:-60}" PORT="${PORT:-8080}" HOST="${HOST:-127.0.0.1:${PORT}}" BEARER="${BEARER:-test-token-aaaaaaaaaaaaaaaaaaaa}" WRK_THREADS="${WRK_THREADS:-8}" WRK_CONNS="${WRK_CONNS:-256}" SAT_RATIO="${SAT_RATIO:-0.70}" # wrk2 target = saturation * SAT_RATIO # --------------------------------------------------------------------------- # Pick server command + lua script # --------------------------------------------------------------------------- case "$SERVER" in urus-mem) SERVER_CMD=("${REPO_ROOT}/target/release/urus-server" "--addr=127.0.0.1:${PORT}" "--store=memory") ;; urus-sqlite) DB="/tmp/urus-bench-${RUN_ID}.sqlite"; rm -f "${DB}"* SERVER_CMD=("${REPO_ROOT}/target/release/urus-server" "--addr=127.0.0.1:${PORT}" "--store=sqlite" "--db-path=${DB}") ;; axum-mem) SERVER_CMD=("${REPO_ROOT}/target/release/axum-server" "--addr=127.0.0.1:${PORT}" "--store=memory") ;; axum-sqlite) DB="/tmp/axum-bench-${RUN_ID}.sqlite"; rm -f "${DB}"* SERVER_CMD=("${REPO_ROOT}/target/release/axum-server" "--addr=127.0.0.1:${PORT}" "--store=sqlite" "--db-path=${DB}") ;; cowboy-mem|cowboy-sqlite) # cowboy-server is a placeholder in this checkout; wire when ready. echo "runner: cowboy backends are not wired yet" >&2 exit 3 ;; *) echo "runner: unknown server '${SERVER}'" >&2; exit 2 ;; esac case "$SCENARIO" in s1) LUA="${REPO_ROOT}/loadgen/s1_ping.lua" ;; s2) LUA="${REPO_ROOT}/loadgen/s2_user_get.lua" ;; s3) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;; s4) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;; # same script, different store *) echo "runner: unknown scenario '${SCENARIO}'" >&2; exit 2 ;; esac # Compatibility check: s4 requires a sqlite-backed server. if [[ "$SCENARIO" == "s4" && "$SERVER" != *-sqlite ]]; then echo "runner: scenario s4 requires a *-sqlite server (got ${SERVER})" >&2 exit 2 fi # --------------------------------------------------------------------------- # Tool checks # --------------------------------------------------------------------------- need() { command -v "$1" >/dev/null || { echo "runner: missing tool: $1" >&2; exit 4; } } need taskset need wrk need wrk2 HAS_PIDSTAT=0; command -v pidstat >/dev/null && HAS_PIDSTAT=1 HAS_JQ=0; command -v jq >/dev/null && HAS_JQ=1 # --------------------------------------------------------------------------- # Verify the server binary actually exists before launching. Without this # check, a missing binary surfaces as "server did not come up" 10 seconds # later, which is misleading. # --------------------------------------------------------------------------- SERVER_BIN="${SERVER_CMD[0]}" if [[ ! -x "$SERVER_BIN" ]]; then echo "runner: server binary not found or not executable: ${SERVER_BIN}" >&2 echo "runner: did you run \`cargo build --release\` in the workspace root?" >&2 exit 5 fi # --------------------------------------------------------------------------- # Boot server (pinned), wait for it to listen # --------------------------------------------------------------------------- SERVER_LOG="${RESULTS_DIR}/server.log" echo "runner: launching server on cores ${SERVER_CPUS}: ${SERVER_CMD[*]}" taskset -c "$SERVER_CPUS" "${SERVER_CMD[@]}" \ > "$SERVER_LOG" 2>&1 & SERVER_PID=$! cleanup() { if kill -0 "$SERVER_PID" 2>/dev/null; then kill "$SERVER_PID" 2>/dev/null || true wait "$SERVER_PID" 2>/dev/null || true fi } trap cleanup EXIT INT TERM # Wait up to 10s for the port to accept. for _ in $(seq 1 100); do if (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then break fi sleep 0.1 done if ! (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then echo "runner: server did not come up; see ${SERVER_LOG}" >&2 exit 5 fi # --------------------------------------------------------------------------- # Wire-level smoke test โ€” fail fast if the routes don't behave. # --------------------------------------------------------------------------- smoke_curl() { local expected="$1"; shift local got got=$(curl -s -o /dev/null -w "%{http_code}" "$@") if [[ "$got" != "$expected" ]]; then echo "runner: smoke fail: expected ${expected} got ${got} for: $*" >&2 exit 6 fi } smoke_curl 200 "http://127.0.0.1:${PORT}/ping" smoke_curl 200 -H "Authorization: Bearer ${BEARER}" \ "http://127.0.0.1:${PORT}/api/v1/users" smoke_curl 401 "http://127.0.0.1:${PORT}/api/v1/users" echo "runner: smoke OK" # --------------------------------------------------------------------------- # Pre-populate store for s2/s3/s4 (s1 doesn't touch the store). # --------------------------------------------------------------------------- # # The s2 / s3 / s4 Lua scripts pick random IDs in [1, 10000]. Against an # empty store every GET-by-id is a 404 fast path on most stacks, which is # *not* what we want to bench. Seed the store with PREPOP=10000 users # (matching the ID range) so GETs hit the actual happy path. This is also # why the bench-spec calls for warm-up: pre-population is part of that. # # For s1 (/ping), skip prepop entirely. PREPOP="${PREPOP:-10000}" if [[ "$SCENARIO" != "s1" && "$PREPOP" -gt 0 ]]; then echo "runner: pre-populating store with ${PREPOP} users" TMP_REQUESTS=$(mktemp) # awk is ~100x faster than a bash for-loop at producing this file. awk -v n="$PREPOP" -v tok="$BEARER" -v port="$PORT" ' BEGIN { for (i = 1; i <= n; i++) { printf "url = http://127.0.0.1:%s/api/v1/users\n", port printf "request = POST\n" printf "header = \"Authorization: Bearer %s\"\n", tok printf "header = \"Content-Type: application/json\"\n" printf "data = \"{\\\"name\\\":\\\"u%d\\\",\\\"email\\\":\\\"u%d@x\\\"}\"\n", i, i if (i < n) printf "next\n" } }' > "$TMP_REQUESTS" # Send all curl output (response bodies + status) to the runner log. # -K does not honor a global -o; the cleanest suppression is shell-level. curl -s -K "$TMP_REQUESTS" >> "$SERVER_LOG" 2>&1 || true rm -f "$TMP_REQUESTS" # Verify list has at least one entry; print count for visibility. N=$(curl -s -H "Authorization: Bearer ${BEARER}" \ "http://127.0.0.1:${PORT}/api/v1/users" | tr ',' '\n' | grep -c '"id"' || echo 0) echo "runner: store now reports ${N} users (list cap is 100)" fi # --------------------------------------------------------------------------- # pidstat sampler (background) โ€” CPU% and RSS at 1Hz # --------------------------------------------------------------------------- STATS_LOG="${RESULTS_DIR}/proc-stats.tsv" echo -e "ts\tcpu_pct\trss_kb" > "$STATS_LOG" ( while kill -0 "$SERVER_PID" 2>/dev/null; do ts=$(date +%s) rss=$(awk '/^VmRSS:/ {print $2}' "/proc/${SERVER_PID}/status" 2>/dev/null || echo 0) if [[ "$HAS_PIDSTAT" == "1" ]]; then cpu=$(pidstat -p "$SERVER_PID" 1 1 2>/dev/null \ | awk '/Average:/ {print $8}' | head -1) else cpu="" fi echo -e "${ts}\t${cpu}\t${rss}" >> "$STATS_LOG" sleep 1 done ) & STATS_PID=$! # --------------------------------------------------------------------------- # Phase 1: saturation probe with wrk (closed-loop peak) # --------------------------------------------------------------------------- echo "runner: probe (${PROBE_SEC}s closed-loop, ${WRK_CONNS} conns)" PROBE_OUT="${RESULTS_DIR}/probe.txt" taskset -c "$LOADGEN_CPUS" \ env HOST="$HOST" BEARER="$BEARER" \ wrk -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${PROBE_SEC}s" --latency \ -s "$LUA" "http://${HOST}" \ > "$PROBE_OUT" 2>&1 || true PROBE_RPS=$(awk '/Requests\/sec:/ {print $2}' "$PROBE_OUT" | head -1) if [[ -z "$PROBE_RPS" ]]; then echo "runner: probe did not report Requests/sec; see ${PROBE_OUT}" >&2 exit 7 fi # floor(PROBE_RPS * SAT_RATIO) as an integer (wrk2 requires int -R) TARGET_RPS=$(awk -v p="$PROBE_RPS" -v r="$SAT_RATIO" \ 'BEGIN { printf("%d", int(p * r)) }') echo "runner: probe RPS=${PROBE_RPS}, target (sustained)=${TARGET_RPS}" # --------------------------------------------------------------------------- # Phase 2: warm-up at target rate (discarded) # --------------------------------------------------------------------------- echo "runner: warm-up (${WARMUP_SEC}s at -R${TARGET_RPS})" taskset -c "$LOADGEN_CPUS" \ env HOST="$HOST" BEARER="$BEARER" \ wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${WARMUP_SEC}s" \ -R"$TARGET_RPS" -s "$LUA" "http://${HOST}" \ > "${RESULTS_DIR}/warmup.txt" 2>&1 || true # --------------------------------------------------------------------------- # Phase 3: measured run with wrk2 (latency-honest) # --------------------------------------------------------------------------- echo "runner: measure (${MEASURE_SEC}s at -R${TARGET_RPS})" MEASURE_OUT="${RESULTS_DIR}/measure.txt" taskset -c "$LOADGEN_CPUS" \ env HOST="$HOST" BEARER="$BEARER" \ wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${MEASURE_SEC}s" \ -R"$TARGET_RPS" --latency \ -s "$LUA" "http://${HOST}" \ > "$MEASURE_OUT" 2>&1 # --------------------------------------------------------------------------- # Phase 4: parse + write result.json # --------------------------------------------------------------------------- # wrk2 output format (excerpt; this regex stack is fragile but works on # upstream wrk2's current build): # # Requests/sec: 149999.74 # Latency Distribution (HdrHistogram - Recorded Latency) # 50.000% 0.50ms # 75.000% 0.61ms # ... # 99.000% 1.83ms # 99.900% 4.21ms # 99.990% 13.50ms # 99.999% 24.50ms # 100.000% 31.50ms extract() { local label="$1" # e.g. "50.000%" awk -v lbl="$label" '$1 == lbl { print $2; exit }' "$MEASURE_OUT" } SUSTAINED_RPS=$(awk '/Requests\/sec:/ {print $2}' "$MEASURE_OUT" | head -1) P50=$(extract "50.000%") P90=$(extract "90.000%") P99=$(extract "99.000%") P999=$(extract "99.900%") MAX=$(extract "100.000%") NON_2XX=$(awk '/Non-2xx or 3xx responses:/ {print $5}' "$MEASURE_OUT" | head -1) # Coerce to a JSON integer; empty (line absent โ†’ no errors) or non-numeric # becomes 0. if ! [[ "$NON_2XX" =~ ^[0-9]+$ ]]; then NON_2XX=0; fi # Best-effort RSS / CPU summary from the sampler MEAN_CPU=$(awk -F'\t' 'NR>1 && $2 != "" { s+=$2; n++ } END { if (n>0) printf("%.1f", s/n); else print "" }' "$STATS_LOG") MAX_RSS=$(awk -F'\t' 'NR>1 { if ($3+0 > m) m=$3 } END { print m+0 }' "$STATS_LOG") cat > "${RESULTS_DIR}/result.json" </dev/null || true wait "$STATS_PID" 2>/dev/null || true echo echo "runner: DONE -> ${RESULTS_DIR}" if [[ "$HAS_JQ" == "1" ]]; then jq . "${RESULTS_DIR}/result.json" else cat "${RESULTS_DIR}/result.json" fi