baseline
This commit is contained in:
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env bash
|
||||
# urus-bench/runner.sh
|
||||
#
|
||||
# Runs one (server, scenario) measurement and saves all artefacts under
|
||||
# results/${run_id}/. The intent is to make a single run trivially
|
||||
# repeatable so the headline table in the bench-spec can be filled in
|
||||
# row by row.
|
||||
#
|
||||
# Usage:
|
||||
# ./runner.sh <server> <scenario> [run_id]
|
||||
#
|
||||
# server:
|
||||
# urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite
|
||||
# scenario:
|
||||
# s1 | s2 | s3 | s4
|
||||
# run_id:
|
||||
# optional; defaults to ${date}-${server}-${scenario}
|
||||
#
|
||||
# Requirements on the host:
|
||||
# - taskset (util-linux)
|
||||
# - wrk2 in PATH (https://github.com/giltene/wrk2)
|
||||
# - wrk in PATH (saturation probe)
|
||||
# - pidstat (sysstat) -- optional, CPU stats only
|
||||
# - jq -- pretty-prints the result.json
|
||||
#
|
||||
# CPU pinning (defaults; override with SERVER_CPUS / LOADGEN_CPUS):
|
||||
# server: taskset -c 0-7
|
||||
# loadgen: taskset -c 8-15
|
||||
#
|
||||
# Probe / measurement times: 30s probe, 30s warm-up, 60s measured (per the
|
||||
# bench-spec §1.5). Override with PROBE_SEC / WARMUP_SEC / MEASURE_SEC.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Args / config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SERVER="${1:-}"
|
||||
SCENARIO="${2:-}"
|
||||
RUN_ID="${3:-$(date +%Y%m%d-%H%M%S)-${SERVER}-${SCENARIO}}"
|
||||
|
||||
if [[ -z "$SERVER" || -z "$SCENARIO" ]]; then
|
||||
echo "usage: $0 <server> <scenario> [run_id]" >&2
|
||||
echo " server: urus-mem | urus-sqlite | axum-mem | axum-sqlite | cowboy-mem | cowboy-sqlite" >&2
|
||||
echo " scenario: s1 | s2 | s3 | s4" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
RESULTS_DIR="${REPO_ROOT}/results/${RUN_ID}"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
SERVER_CPUS="${SERVER_CPUS:-0-7}"
|
||||
LOADGEN_CPUS="${LOADGEN_CPUS:-8-15}"
|
||||
PROBE_SEC="${PROBE_SEC:-30}"
|
||||
WARMUP_SEC="${WARMUP_SEC:-30}"
|
||||
MEASURE_SEC="${MEASURE_SEC:-60}"
|
||||
PORT="${PORT:-8080}"
|
||||
HOST="${HOST:-127.0.0.1:${PORT}}"
|
||||
BEARER="${BEARER:-test-token-aaaaaaaaaaaaaaaaaaaa}"
|
||||
WRK_THREADS="${WRK_THREADS:-8}"
|
||||
WRK_CONNS="${WRK_CONNS:-256}"
|
||||
SAT_RATIO="${SAT_RATIO:-0.70}" # wrk2 target = saturation * SAT_RATIO
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pick server command + lua script
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
case "$SERVER" in
|
||||
urus-mem)
|
||||
SERVER_CMD=("${REPO_ROOT}/target/release/urus-server"
|
||||
"--addr=127.0.0.1:${PORT}" "--store=memory") ;;
|
||||
urus-sqlite)
|
||||
DB="/tmp/urus-bench-${RUN_ID}.sqlite"; rm -f "${DB}"*
|
||||
SERVER_CMD=("${REPO_ROOT}/target/release/urus-server"
|
||||
"--addr=127.0.0.1:${PORT}" "--store=sqlite"
|
||||
"--db-path=${DB}") ;;
|
||||
axum-mem)
|
||||
SERVER_CMD=("${REPO_ROOT}/target/release/axum-server"
|
||||
"--addr=127.0.0.1:${PORT}" "--store=memory") ;;
|
||||
axum-sqlite)
|
||||
DB="/tmp/axum-bench-${RUN_ID}.sqlite"; rm -f "${DB}"*
|
||||
SERVER_CMD=("${REPO_ROOT}/target/release/axum-server"
|
||||
"--addr=127.0.0.1:${PORT}" "--store=sqlite"
|
||||
"--db-path=${DB}") ;;
|
||||
cowboy-mem|cowboy-sqlite)
|
||||
# cowboy-server is a placeholder in this checkout; wire when ready.
|
||||
echo "runner: cowboy backends are not wired yet" >&2
|
||||
exit 3 ;;
|
||||
*)
|
||||
echo "runner: unknown server '${SERVER}'" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
case "$SCENARIO" in
|
||||
s1) LUA="${REPO_ROOT}/loadgen/s1_ping.lua" ;;
|
||||
s2) LUA="${REPO_ROOT}/loadgen/s2_user_get.lua" ;;
|
||||
s3) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;;
|
||||
s4) LUA="${REPO_ROOT}/loadgen/s3_mixed.lua" ;; # same script, different store
|
||||
*) echo "runner: unknown scenario '${SCENARIO}'" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# Compatibility check: s4 requires a sqlite-backed server.
|
||||
if [[ "$SCENARIO" == "s4" && "$SERVER" != *-sqlite ]]; then
|
||||
echo "runner: scenario s4 requires a *-sqlite server (got ${SERVER})" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null || { echo "runner: missing tool: $1" >&2; exit 4; }
|
||||
}
|
||||
need taskset
|
||||
need wrk
|
||||
need wrk2
|
||||
|
||||
HAS_PIDSTAT=0; command -v pidstat >/dev/null && HAS_PIDSTAT=1
|
||||
HAS_JQ=0; command -v jq >/dev/null && HAS_JQ=1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verify the server binary actually exists before launching. Without this
|
||||
# check, a missing binary surfaces as "server did not come up" 10 seconds
|
||||
# later, which is misleading.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SERVER_BIN="${SERVER_CMD[0]}"
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
echo "runner: server binary not found or not executable: ${SERVER_BIN}" >&2
|
||||
echo "runner: did you run \`cargo build --release\` in the workspace root?" >&2
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boot server (pinned), wait for it to listen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SERVER_LOG="${RESULTS_DIR}/server.log"
|
||||
echo "runner: launching server on cores ${SERVER_CPUS}: ${SERVER_CMD[*]}"
|
||||
taskset -c "$SERVER_CPUS" "${SERVER_CMD[@]}" \
|
||||
> "$SERVER_LOG" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
if kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Wait up to 10s for the port to accept.
|
||||
for _ in $(seq 1 100); do
|
||||
if (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
if ! (echo > /dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then
|
||||
echo "runner: server did not come up; see ${SERVER_LOG}" >&2
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wire-level smoke test — fail fast if the routes don't behave.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
smoke_curl() {
|
||||
local expected="$1"; shift
|
||||
local got
|
||||
got=$(curl -s -o /dev/null -w "%{http_code}" "$@")
|
||||
if [[ "$got" != "$expected" ]]; then
|
||||
echo "runner: smoke fail: expected ${expected} got ${got} for: $*" >&2
|
||||
exit 6
|
||||
fi
|
||||
}
|
||||
|
||||
smoke_curl 200 "http://127.0.0.1:${PORT}/ping"
|
||||
smoke_curl 200 -H "Authorization: Bearer ${BEARER}" \
|
||||
"http://127.0.0.1:${PORT}/api/v1/users"
|
||||
smoke_curl 401 "http://127.0.0.1:${PORT}/api/v1/users"
|
||||
|
||||
echo "runner: smoke OK"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-populate store for s2/s3/s4 (s1 doesn't touch the store).
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The s2 / s3 / s4 Lua scripts pick random IDs in [1, 10000]. Against an
|
||||
# empty store every GET-by-id is a 404 fast path on most stacks, which is
|
||||
# *not* what we want to bench. Seed the store with PREPOP=10000 users
|
||||
# (matching the ID range) so GETs hit the actual happy path. This is also
|
||||
# why the bench-spec calls for warm-up: pre-population is part of that.
|
||||
#
|
||||
# For s1 (/ping), skip prepop entirely.
|
||||
|
||||
PREPOP="${PREPOP:-10000}"
|
||||
if [[ "$SCENARIO" != "s1" && "$PREPOP" -gt 0 ]]; then
|
||||
echo "runner: pre-populating store with ${PREPOP} users"
|
||||
TMP_REQUESTS=$(mktemp)
|
||||
# awk is ~100x faster than a bash for-loop at producing this file.
|
||||
awk -v n="$PREPOP" -v tok="$BEARER" -v port="$PORT" '
|
||||
BEGIN {
|
||||
for (i = 1; i <= n; i++) {
|
||||
printf "url = http://127.0.0.1:%s/api/v1/users\n", port
|
||||
printf "request = POST\n"
|
||||
printf "header = \"Authorization: Bearer %s\"\n", tok
|
||||
printf "header = \"Content-Type: application/json\"\n"
|
||||
printf "data = \"{\\\"name\\\":\\\"u%d\\\",\\\"email\\\":\\\"u%d@x\\\"}\"\n", i, i
|
||||
if (i < n) printf "next\n"
|
||||
}
|
||||
}' > "$TMP_REQUESTS"
|
||||
# Send all curl output (response bodies + status) to the runner log.
|
||||
# -K does not honor a global -o; the cleanest suppression is shell-level.
|
||||
curl -s -K "$TMP_REQUESTS" >> "$SERVER_LOG" 2>&1 || true
|
||||
rm -f "$TMP_REQUESTS"
|
||||
# Verify list has at least one entry; print count for visibility.
|
||||
N=$(curl -s -H "Authorization: Bearer ${BEARER}" \
|
||||
"http://127.0.0.1:${PORT}/api/v1/users" | tr ',' '\n' | grep -c '"id"' || echo 0)
|
||||
echo "runner: store now reports ${N} users (list cap is 100)"
|
||||
fi
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pidstat sampler (background) — CPU% and RSS at 1Hz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
STATS_LOG="${RESULTS_DIR}/proc-stats.tsv"
|
||||
echo -e "ts\tcpu_pct\trss_kb" > "$STATS_LOG"
|
||||
(
|
||||
while kill -0 "$SERVER_PID" 2>/dev/null; do
|
||||
ts=$(date +%s)
|
||||
rss=$(awk '/^VmRSS:/ {print $2}' "/proc/${SERVER_PID}/status" 2>/dev/null || echo 0)
|
||||
if [[ "$HAS_PIDSTAT" == "1" ]]; then
|
||||
cpu=$(pidstat -p "$SERVER_PID" 1 1 2>/dev/null \
|
||||
| awk '/Average:/ {print $8}' | head -1)
|
||||
else
|
||||
cpu=""
|
||||
fi
|
||||
echo -e "${ts}\t${cpu}\t${rss}" >> "$STATS_LOG"
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
STATS_PID=$!
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: saturation probe with wrk (closed-loop peak)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo "runner: probe (${PROBE_SEC}s closed-loop, ${WRK_CONNS} conns)"
|
||||
PROBE_OUT="${RESULTS_DIR}/probe.txt"
|
||||
taskset -c "$LOADGEN_CPUS" \
|
||||
env HOST="$HOST" BEARER="$BEARER" \
|
||||
wrk -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${PROBE_SEC}s" --latency \
|
||||
-s "$LUA" "http://${HOST}" \
|
||||
> "$PROBE_OUT" 2>&1 || true
|
||||
|
||||
PROBE_RPS=$(awk '/Requests\/sec:/ {print $2}' "$PROBE_OUT" | head -1)
|
||||
if [[ -z "$PROBE_RPS" ]]; then
|
||||
echo "runner: probe did not report Requests/sec; see ${PROBE_OUT}" >&2
|
||||
exit 7
|
||||
fi
|
||||
|
||||
# floor(PROBE_RPS * SAT_RATIO) as an integer (wrk2 requires int -R)
|
||||
TARGET_RPS=$(awk -v p="$PROBE_RPS" -v r="$SAT_RATIO" \
|
||||
'BEGIN { printf("%d", int(p * r)) }')
|
||||
echo "runner: probe RPS=${PROBE_RPS}, target (sustained)=${TARGET_RPS}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: warm-up at target rate (discarded)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo "runner: warm-up (${WARMUP_SEC}s at -R${TARGET_RPS})"
|
||||
taskset -c "$LOADGEN_CPUS" \
|
||||
env HOST="$HOST" BEARER="$BEARER" \
|
||||
wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${WARMUP_SEC}s" \
|
||||
-R"$TARGET_RPS" -s "$LUA" "http://${HOST}" \
|
||||
> "${RESULTS_DIR}/warmup.txt" 2>&1 || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: measured run with wrk2 (latency-honest)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo "runner: measure (${MEASURE_SEC}s at -R${TARGET_RPS})"
|
||||
MEASURE_OUT="${RESULTS_DIR}/measure.txt"
|
||||
taskset -c "$LOADGEN_CPUS" \
|
||||
env HOST="$HOST" BEARER="$BEARER" \
|
||||
wrk2 -t"$WRK_THREADS" -c"$WRK_CONNS" -d"${MEASURE_SEC}s" \
|
||||
-R"$TARGET_RPS" --latency \
|
||||
-s "$LUA" "http://${HOST}" \
|
||||
> "$MEASURE_OUT" 2>&1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: parse + write result.json
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# wrk2 output format (excerpt; this regex stack is fragile but works on
|
||||
# upstream wrk2's current build):
|
||||
#
|
||||
# Requests/sec: 149999.74
|
||||
# Latency Distribution (HdrHistogram - Recorded Latency)
|
||||
# 50.000% 0.50ms
|
||||
# 75.000% 0.61ms
|
||||
# ...
|
||||
# 99.000% 1.83ms
|
||||
# 99.900% 4.21ms
|
||||
# 99.990% 13.50ms
|
||||
# 99.999% 24.50ms
|
||||
# 100.000% 31.50ms
|
||||
|
||||
extract() {
|
||||
local label="$1" # e.g. "50.000%"
|
||||
awk -v lbl="$label" '$1 == lbl { print $2; exit }' "$MEASURE_OUT"
|
||||
}
|
||||
|
||||
SUSTAINED_RPS=$(awk '/Requests\/sec:/ {print $2}' "$MEASURE_OUT" | head -1)
|
||||
P50=$(extract "50.000%")
|
||||
P90=$(extract "90.000%")
|
||||
P99=$(extract "99.000%")
|
||||
P999=$(extract "99.900%")
|
||||
MAX=$(extract "100.000%")
|
||||
NON_2XX=$(awk '/Non-2xx or 3xx responses:/ {print $5}' "$MEASURE_OUT" | head -1)
|
||||
# Coerce to a JSON integer; empty (line absent → no errors) or non-numeric
|
||||
# becomes 0.
|
||||
if ! [[ "$NON_2XX" =~ ^[0-9]+$ ]]; then NON_2XX=0; fi
|
||||
|
||||
# Best-effort RSS / CPU summary from the sampler
|
||||
MEAN_CPU=$(awk -F'\t' 'NR>1 && $2 != "" { s+=$2; n++ } END { if (n>0) printf("%.1f", s/n); else print "" }' "$STATS_LOG")
|
||||
MAX_RSS=$(awk -F'\t' 'NR>1 { if ($3+0 > m) m=$3 } END { print m+0 }' "$STATS_LOG")
|
||||
|
||||
cat > "${RESULTS_DIR}/result.json" <<JSON
|
||||
{
|
||||
"run_id": "${RUN_ID}",
|
||||
"server": "${SERVER}",
|
||||
"scenario": "${SCENARIO}",
|
||||
"probe_rps": ${PROBE_RPS},
|
||||
"target_rps": ${TARGET_RPS},
|
||||
"sustained_rps": ${SUSTAINED_RPS:-0},
|
||||
"latency": {
|
||||
"p50": "${P50}",
|
||||
"p90": "${P90}",
|
||||
"p99": "${P99}",
|
||||
"p999": "${P999}",
|
||||
"max": "${MAX}"
|
||||
},
|
||||
"non_2xx": ${NON_2XX},
|
||||
"mean_cpu_pct": "${MEAN_CPU}",
|
||||
"max_rss_kb": ${MAX_RSS}
|
||||
}
|
||||
JSON
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Done
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
kill "$STATS_PID" 2>/dev/null || true
|
||||
wait "$STATS_PID" 2>/dev/null || true
|
||||
|
||||
echo
|
||||
echo "runner: DONE -> ${RESULTS_DIR}"
|
||||
if [[ "$HAS_JQ" == "1" ]]; then
|
||||
jq . "${RESULTS_DIR}/result.json"
|
||||
else
|
||||
cat "${RESULTS_DIR}/result.json"
|
||||
fi
|
||||
Reference in New Issue
Block a user