368 lines
13 KiB
Bash
Executable File
368 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# urus-bench/runner.sh
|
|
#
|
|
# Runs one (server, scenario) measurement and saves all artefacts under
|
|
# results/${run_id}/. The intent is to make a single run trivially
|
|
# repeatable so the headline table in the bench-spec can be filled in
|
|
# row by row.
|
|
#
|
|
# Usage:
|
|
# ./runner.sh <server> <scenario> [run_id]
|
|
#
|
|
# server:
|
|
# urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite
|
|
# scenario:
|
|
# s1 | s2 | s3 | s4
|
|
# run_id:
|
|
# optional; defaults to ${date}-${server}-${scenario}
|
|
#
|
|
# Requirements on the host:
|
|
# - taskset (util-linux)
|
|
# - wrk2 in PATH (https://github.com/giltene/wrk2)
|
|
# - wrk in PATH (saturation probe)
|
|
# - pidstat (sysstat) -- optional, CPU stats only
|
|
# - jq -- pretty-prints the result.json
|
|
#
|
|
# CPU pinning (defaults; override with SERVER_CPUS / LOADGEN_CPUS):
|
|
# server: taskset -c 0-7
|
|
# loadgen: taskset -c 8-15
|
|
#
|
|
# Probe / measurement times: 30s probe, 30s warm-up, 60s measured (per the
|
|
# bench-spec §1.5). Override with PROBE_SEC / WARMUP_SEC / MEASURE_SEC.
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Args / config
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SERVER="${1:-}"
|
|
SCENARIO="${2:-}"
|
|
RUN_ID="${3:-$(date +%Y%m%d-%H%M%S)-${SERVER}-${SCENARIO}}"
|
|
|
|
if [[ -z "$SERVER" || -z "$SCENARIO" ]]; then
|
|
echo "usage: $0 <server> <scenario> [run_id]" >&2
|
|
echo " server: urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite" >&2
|
|
echo " scenario: s1 | s2 | s3 | s4" >&2
|
|
exit 2
|
|
fi
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
RESULTS_DIR="${REPO_ROOT}/results/${RUN_ID}"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
SERVER_CPUS="${SERVER_CPUS:-0-7}"
|
|
LOADGEN_CPUS="${LOADGEN_CPUS:-8-15}"
|
|
PROBE_SEC="${PROBE_SEC:-30}"
|
|
WARMUP_SEC="${WARMUP_SEC:-30}"
|
|
MEASURE_SEC="${MEASURE_SEC:-60}"
|
|
PORT="${PORT:-8080}"
|
|
HOST="${HOST:-127.0.0.1:${PORT}}"
|
|
BEARER="${BEARER:-test-token-aaaaaaaaaaaaaaaaaaaa}"
|
|
WRK_THREADS="${WRK_THREADS:-8}"
|
|
WRK_CONNS="${WRK_CONNS:-256}"
|
|
SAT_RATIO="${SAT_RATIO:-0.70}" # wrk2 target = saturation * SAT_RATIO
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pick server command + lua script
|
|
# ---------------------------------------------------------------------------
|
|
|
|
case "$SERVER" in
|
|
urus-mem)
|
|
SERVER_CMD=("${REPO_ROOT}/target/release/urus-server"
|
|
"--addr=127.0.0.1:${PORT}" "--store=memory") ;;
|
|
urus-sqlite)
|
|
DB="/tmp/urus-bench-${RUN_ID}.sqlite"; rm -f "${DB}"*
|
|
SERVER_CMD=("${REPO_ROOT}/target/release/urus-server"
|
|
"--addr=127.0.0.1:${PORT}" "--store=sqlite"
|
|
"--db-path=${DB}") ;;
|
|
axum-mem)
|
|
SERVER_CMD=("${REPO_ROOT}/target/release/axum-server"
|
|
"--addr=127.0.0.1:${PORT}" "--store=memory") ;;
|
|
axum-sqlite)
|
|
DB="/tmp/axum-bench-${RUN_ID}.sqlite"; rm -f "${DB}"*
|
|
SERVER_CMD=("${REPO_ROOT}/target/release/axum-server"
|
|
"--addr=127.0.0.1:${PORT}" "--store=sqlite"
|
|
"--db-path=${DB}") ;;
|
|
cowboy-mem|cowboy-sqlite)
|
|
# cowboy-server is a placeholder in this checkout; wire when ready.
|
|
echo "runner: cowboy backends are not wired yet" >&2
|
|
exit 3 ;;
|
|
*)
|
|
echo "runner: unknown server '${SERVER}'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
case "$SCENARIO" in
|
|
s1) LUA="${REPO_ROOT}/loadgen/s1_ping.lua" ;;
|
|
s2) LUA="${REPO_ROOT}/loadgen/s2_user_get.lua" ;;
|
|
s3) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;;
|
|
s4) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;; # same script, different store
|
|
*) echo "runner: unknown scenario '${SCENARIO}'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
# Compatibility check: s4 requires a sqlite-backed server.
|
|
if [[ "$SCENARIO" == "s4" && "$SERVER" != *-sqlite ]]; then
|
|
echo "runner: scenario s4 requires a *-sqlite server (got ${SERVER})" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
need() {
|
|
command -v "$1" >/dev/null || { echo "runner: missing tool: $1" >&2; exit 4; }
|
|
}
|
|
need taskset
|
|
need wrk
|
|
need wrk2
|
|
|
|
HAS_PIDSTAT=0; command -v pidstat >/dev/null && HAS_PIDSTAT=1
|
|
HAS_JQ=0; command -v jq >/dev/null && HAS_JQ=1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Verify the server binary actually exists before launching. Without this
|
|
# check, a missing binary surfaces as "server did not come up" 10 seconds
|
|
# later, which is misleading.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SERVER_BIN="${SERVER_CMD[0]}"
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "runner: server binary not found or not executable: ${SERVER_BIN}" >&2
|
|
echo "runner: did you run \`cargo build --release\` in the workspace root?" >&2
|
|
exit 5
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Boot server (pinned), wait for it to listen
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SERVER_LOG="${RESULTS_DIR}/server.log"
|
|
echo "runner: launching server on cores ${SERVER_CPUS}: ${SERVER_CMD[*]}"
|
|
taskset -c "$SERVER_CPUS" "${SERVER_CMD[@]}" \
|
|
> "$SERVER_LOG" 2>&1 &
|
|
SERVER_PID=$!
|
|
|
|
cleanup() {
|
|
if kill -0 "$SERVER_PID" 2>/dev/null; then
|
|
kill "$SERVER_PID" 2>/dev/null || true
|
|
wait "$SERVER_PID" 2>/dev/null || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Wait up to 10s for the port to accept.
|
|
for _ in $(seq 1 100); do
|
|
if (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
if ! (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then
|
|
echo "runner: server did not come up; see ${SERVER_LOG}" >&2
|
|
exit 5
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Wire-level smoke test — fail fast if the routes don't behave.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
smoke_curl() {
|
|
local expected="$1"; shift
|
|
local got
|
|
got=$(curl -s -o /dev/null -w "%{http_code}" "$@")
|
|
if [[ "$got" != "$expected" ]]; then
|
|
echo "runner: smoke fail: expected ${expected} got ${got} for: $*" >&2
|
|
exit 6
|
|
fi
|
|
}
|
|
|
|
smoke_curl 200 "http://127.0.0.1:${PORT}/ping"
|
|
smoke_curl 200 -H "Authorization: Bearer ${BEARER}" \
|
|
"http://127.0.0.1:${PORT}/api/v1/users"
|
|
smoke_curl 401 "http://127.0.0.1:${PORT}/api/v1/users"
|
|
|
|
echo "runner: smoke OK"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pre-populate store for s2/s3/s4 (s1 doesn't touch the store).
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# The s2 / s3 / s4 Lua scripts pick random IDs in [1, 10000]. Against an
|
|
# empty store every GET-by-id is a 404 fast path on most stacks, which is
|
|
# *not* what we want to bench. Seed the store with PREPOP=10000 users
|
|
# (matching the ID range) so GETs hit the actual happy path. This is also
|
|
# why the bench-spec calls for warm-up: pre-population is part of that.
|
|
#
|
|
# For s1 (/ping), skip prepop entirely.
|
|
|
|
PREPOP="${PREPOP:-10000}"
|
|
if [[ "$SCENARIO" != "s1" && "$PREPOP" -gt 0 ]]; then
|
|
echo "runner: pre-populating store with ${PREPOP} users"
|
|
TMP_REQUESTS=$(mktemp)
|
|
# awk is ~100x faster than a bash for-loop at producing this file.
|
|
awk -v n="$PREPOP" -v tok="$BEARER" -v port="$PORT" '
|
|
BEGIN {
|
|
for (i = 1; i <= n; i++) {
|
|
printf "url = http://127.0.0.1:%s/api/v1/users\n", port
|
|
printf "request = POST\n"
|
|
printf "header = \"Authorization: Bearer %s\"\n", tok
|
|
printf "header = \"Content-Type: application/json\"\n"
|
|
printf "data = \"{\\\"name\\\":\\\"u%d\\\",\\\"email\\\":\\\"u%d@x\\\"}\"\n", i, i
|
|
if (i < n) printf "next\n"
|
|
}
|
|
}' > "$TMP_REQUESTS"
|
|
# Send all curl output (response bodies + status) to the runner log.
|
|
# -K does not honor a global -o; the cleanest suppression is shell-level.
|
|
curl -s -K "$TMP_REQUESTS" >> "$SERVER_LOG" 2>&1 || true
|
|
rm -f "$TMP_REQUESTS"
|
|
# Verify list has at least one entry; print count for visibility.
|
|
N=$(curl -s -H "Authorization: Bearer ${BEARER}" \
|
|
"http://127.0.0.1:${PORT}/api/v1/users" | tr ',' '\n' | grep -c '"id"' || echo 0)
|
|
echo "runner: store now reports ${N} users (list cap is 100)"
|
|
fi
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# pidstat sampler (background) — CPU% and RSS at 1Hz
|
|
# ---------------------------------------------------------------------------
|
|
|
|
STATS_LOG="${RESULTS_DIR}/proc-stats.tsv"
|
|
echo -e "ts\tcpu_pct\trss_kb" > "$STATS_LOG"
|
|
(
|
|
while kill -0 "$SERVER_PID" 2>/dev/null; do
|
|
ts=$(date +%s)
|
|
rss=$(awk '/^VmRSS:/ {print $2}' "/proc/${SERVER_PID}/status" 2>/dev/null || echo 0)
|
|
if [[ "$HAS_PIDSTAT" == "1" ]]; then
|
|
cpu=$(pidstat -p "$SERVER_PID" 1 1 2>/dev/null \
|
|
| awk '/Average:/ {print $8}' | head -1)
|
|
else
|
|
cpu=""
|
|
fi
|
|
echo -e "${ts}\t${cpu}\t${rss}" >> "$STATS_LOG"
|
|
sleep 1
|
|
done
|
|
) &
|
|
STATS_PID=$!
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 1: saturation probe with wrk (closed-loop peak)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo "runner: probe (${PROBE_SEC}s closed-loop, ${WRK_CONNS} conns)"
|
|
PROBE_OUT="${RESULTS_DIR}/probe.txt"
|
|
taskset -c "$LOADGEN_CPUS" \
|
|
env HOST="$HOST" BEARER="$BEARER" \
|
|
wrk -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${PROBE_SEC}s" --latency \
|
|
-s "$LUA" "http://${HOST}" \
|
|
> "$PROBE_OUT" 2>&1 || true
|
|
|
|
PROBE_RPS=$(awk '/Requests\/sec:/ {print $2}' "$PROBE_OUT" | head -1)
|
|
if [[ -z "$PROBE_RPS" ]]; then
|
|
echo "runner: probe did not report Requests/sec; see ${PROBE_OUT}" >&2
|
|
exit 7
|
|
fi
|
|
|
|
# floor(PROBE_RPS * SAT_RATIO) as an integer (wrk2 requires int -R)
|
|
TARGET_RPS=$(awk -v p="$PROBE_RPS" -v r="$SAT_RATIO" \
|
|
'BEGIN { printf("%d", int(p * r)) }')
|
|
echo "runner: probe RPS=${PROBE_RPS}, target (sustained)=${TARGET_RPS}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 2: warm-up at target rate (discarded)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo "runner: warm-up (${WARMUP_SEC}s at -R${TARGET_RPS})"
|
|
taskset -c "$LOADGEN_CPUS" \
|
|
env HOST="$HOST" BEARER="$BEARER" \
|
|
wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${WARMUP_SEC}s" \
|
|
-R"$TARGET_RPS" -s "$LUA" "http://${HOST}" \
|
|
> "${RESULTS_DIR}/warmup.txt" 2>&1 || true
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 3: measured run with wrk2 (latency-honest)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo "runner: measure (${MEASURE_SEC}s at -R${TARGET_RPS})"
|
|
MEASURE_OUT="${RESULTS_DIR}/measure.txt"
|
|
taskset -c "$LOADGEN_CPUS" \
|
|
env HOST="$HOST" BEARER="$BEARER" \
|
|
wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${MEASURE_SEC}s" \
|
|
-R"$TARGET_RPS" --latency \
|
|
-s "$LUA" "http://${HOST}" \
|
|
> "$MEASURE_OUT" 2>&1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Phase 4: parse + write result.json
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# wrk2 output format (excerpt; this regex stack is fragile but works on
|
|
# upstream wrk2's current build):
|
|
#
|
|
# Requests/sec: 149999.74
|
|
# Latency Distribution (HdrHistogram - Recorded Latency)
|
|
# 50.000% 0.50ms
|
|
# 75.000% 0.61ms
|
|
# ...
|
|
# 99.000% 1.83ms
|
|
# 99.900% 4.21ms
|
|
# 99.990% 13.50ms
|
|
# 99.999% 24.50ms
|
|
# 100.000% 31.50ms
|
|
|
|
extract() {
|
|
local label="$1" # e.g. "50.000%"
|
|
awk -v lbl="$label" '$1 == lbl { print $2; exit }' "$MEASURE_OUT"
|
|
}
|
|
|
|
SUSTAINED_RPS=$(awk '/Requests\/sec:/ {print $2}' "$MEASURE_OUT" | head -1)
|
|
P50=$(extract "50.000%")
|
|
P90=$(extract "90.000%")
|
|
P99=$(extract "99.000%")
|
|
P999=$(extract "99.900%")
|
|
MAX=$(extract "100.000%")
|
|
NON_2XX=$(awk '/Non-2xx or 3xx responses:/ {print $5}' "$MEASURE_OUT" | head -1)
|
|
# Coerce to a JSON integer; empty (line absent → no errors) or non-numeric
|
|
# becomes 0.
|
|
if ! [[ "$NON_2XX" =~ ^[0-9]+$ ]]; then NON_2XX=0; fi
|
|
|
|
# Best-effort RSS / CPU summary from the sampler
|
|
MEAN_CPU=$(awk -F'\t' 'NR>1 && $2 != "" { s+=$2; n++ } END { if (n>0) printf("%.1f", s/n); else print "" }' "$STATS_LOG")
|
|
MAX_RSS=$(awk -F'\t' 'NR>1 { if ($3+0 > m) m=$3 } END { print m+0 }' "$STATS_LOG")
|
|
|
|
cat > "${RESULTS_DIR}/result.json" <<JSON
|
|
{
|
|
"run_id": "${RUN_ID}",
|
|
"server": "${SERVER}",
|
|
"scenario": "${SCENARIO}",
|
|
"probe_rps": ${PROBE_RPS},
|
|
"target_rps": ${TARGET_RPS},
|
|
"sustained_rps": ${SUSTAINED_RPS:-0},
|
|
"latency": {
|
|
"p50": "${P50}",
|
|
"p90": "${P90}",
|
|
"p99": "${P99}",
|
|
"p999": "${P999}",
|
|
"max": "${MAX}"
|
|
},
|
|
"non_2xx": ${NON_2XX},
|
|
"mean_cpu_pct": "${MEAN_CPU}",
|
|
"max_rss_kb": ${MAX_RSS}
|
|
}
|
|
JSON
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Done
|
|
# ---------------------------------------------------------------------------
|
|
|
|
kill "$STATS_PID" 2>/dev/null || true
|
|
wait "$STATS_PID" 2>/dev/null || true
|
|
|
|
echo
|
|
echo "runner: DONE -> ${RESULTS_DIR}"
|
|
if [[ "$HAS_JQ" == "1" ]]; then
|
|
jq . "${RESULTS_DIR}/result.json"
|
|
else
|
|
cat "${RESULTS_DIR}/result.json"
|
|
fi
|