baseline
This commit is contained in:
Executable
+411
@@ -0,0 +1,411 @@
|
||||
#!/usr/bin/env bash
|
||||
# urus-bench/run_all.sh
|
||||
#
|
||||
# Runs the full (server, scenario) matrix and writes a summary.md.
|
||||
#
|
||||
# Default matrix (8 runs):
|
||||
# urus-mem × s1, s2, s3
|
||||
# axum-mem × s1, s2, s3
|
||||
# urus-sqlite × s4
|
||||
# axum-sqlite × s4
|
||||
#
|
||||
# Cowboy is not built; skipped.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_all.sh # full matrix, defaults
|
||||
# ./run_all.sh --quick # short timings (15s+15s+15s)
|
||||
# ./run_all.sh --scenarios=s1,s2 # subset of scenarios
|
||||
# ./run_all.sh --servers=urus-mem # subset of servers
|
||||
# ./run_all.sh --batch=foo # batch label (default: timestamp)
|
||||
#
|
||||
# Per-run env vars (PROBE_SEC, WARMUP_SEC, MEASURE_SEC, PREPOP, SERVER_CPUS,
|
||||
# LOADGEN_CPUS, WRK_CONNS, WRK_THREADS, SAT_RATIO) are passed through to
|
||||
# runner.sh; --quick is just a preset for the timings.
|
||||
#
|
||||
# Output:
|
||||
# results/<batch>/ batch dir
|
||||
# <server>-<scenario>/ per-run dir (from runner.sh)
|
||||
# summary.md aggregated headline table
|
||||
# index.json machine-readable index of all runs
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ALL_SERVERS_MEM=(urus-mem axum-mem)
|
||||
ALL_SERVERS_SQLITE=(urus-sqlite axum-sqlite)
|
||||
ALL_SCENARIOS_MEM=(s1 s2 s3)
|
||||
ALL_SCENARIOS_SQLITE=(s4)
|
||||
|
||||
SERVERS_FILTER=""
|
||||
SCENARIOS_FILTER=""
|
||||
BATCH=""
|
||||
QUICK=0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Arg parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--quick) QUICK=1 ;;
|
||||
--servers=*) SERVERS_FILTER="${arg#*=}" ;;
|
||||
--scenarios=*) SCENARIOS_FILTER="${arg#*=}" ;;
|
||||
--batch=*) BATCH="${arg#*=}" ;;
|
||||
-h|--help)
|
||||
sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'
|
||||
exit 0 ;;
|
||||
*)
|
||||
echo "run_all: unknown arg: $arg" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$QUICK" == "1" ]]; then
|
||||
export PROBE_SEC="${PROBE_SEC:-15}"
|
||||
export WARMUP_SEC="${WARMUP_SEC:-15}"
|
||||
export MEASURE_SEC="${MEASURE_SEC:-15}"
|
||||
fi
|
||||
|
||||
BATCH="${BATCH:-$(date +%Y%m%d-%H%M%S)}"
|
||||
BATCH_DIR="${REPO_ROOT}/results/${BATCH}"
|
||||
mkdir -p "$BATCH_DIR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build the matrix
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# A run is a pair "<server> <scenario>". We compute the full matrix, then
|
||||
# apply filters. s1/s2/s3 always pair with the memory backends; s4 only
|
||||
# pairs with sqlite backends.
|
||||
|
||||
contains() {
|
||||
# contains "needle" "csv-list" → 0 if needle in csv (or csv empty), else 1
|
||||
local needle="$1" haystack="$2"
|
||||
[[ -z "$haystack" ]] && return 0
|
||||
local IFS=,
|
||||
for item in $haystack; do
|
||||
[[ "$item" == "$needle" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
runs=()
|
||||
for srv in "${ALL_SERVERS_MEM[@]}"; do
|
||||
contains "$srv" "$SERVERS_FILTER" || continue
|
||||
for sc in "${ALL_SCENARIOS_MEM[@]}"; do
|
||||
contains "$sc" "$SCENARIOS_FILTER" || continue
|
||||
runs+=("$srv $sc")
|
||||
done
|
||||
done
|
||||
for srv in "${ALL_SERVERS_SQLITE[@]}"; do
|
||||
contains "$srv" "$SERVERS_FILTER" || continue
|
||||
for sc in "${ALL_SCENARIOS_SQLITE[@]}"; do
|
||||
contains "$sc" "$SCENARIOS_FILTER" || continue
|
||||
runs+=("$srv $sc")
|
||||
done
|
||||
done
|
||||
|
||||
if [[ "${#runs[@]}" -eq 0 ]]; then
|
||||
echo "run_all: empty matrix (filters too narrow?)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build binaries up front
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# We always rebuild in --release. If urus-server or axum-server is not
|
||||
# requested, cargo will still build the workspace, but that's fine — a
|
||||
# 1-minute one-time cost beats a confusing "binary not found" mid-batch.
|
||||
|
||||
echo "run_all: building binaries (release)..."
|
||||
( cd "$REPO_ROOT" && cargo build --release ) || {
|
||||
echo "run_all: cargo build failed" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run the matrix
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
total="${#runs[@]}"
|
||||
echo
|
||||
echo "run_all: batch=${BATCH}, ${total} run(s):"
|
||||
i=0
|
||||
for r in "${runs[@]}"; do
|
||||
i=$((i + 1))
|
||||
printf " [%d/%d] %s\n" "$i" "$total" "$r"
|
||||
done
|
||||
echo
|
||||
|
||||
i=0
|
||||
failed=()
|
||||
succeeded=()
|
||||
START_TS=$(date +%s)
|
||||
|
||||
# Per-run port offset so concurrent stale processes (if any) don't collide.
|
||||
BASE_PORT="${BASE_PORT:-8080}"
|
||||
|
||||
for r in "${runs[@]}"; do
|
||||
i=$((i + 1))
|
||||
set -- $r
|
||||
SERVER="$1"; SCENARIO="$2"
|
||||
RUN_ID="${SERVER}-${SCENARIO}"
|
||||
RUN_DIR="${BATCH_DIR}/${RUN_ID}"
|
||||
|
||||
printf '\n========== [%d/%d] %-12s %-3s ==========\n' \
|
||||
"$i" "$total" "$SERVER" "$SCENARIO"
|
||||
|
||||
# Each run gets its own port so a hung server from a previous run can't
|
||||
# silently keep answering on the default port.
|
||||
PORT=$((BASE_PORT + i))
|
||||
|
||||
# Each run gets a unique results dir under the batch dir. runner.sh
|
||||
# writes to results/<run_id>/ at the repo root by default — we pass the
|
||||
# run_id but also need it to land under the batch dir. Cheapest fix:
|
||||
# let runner write to its default location, then move it under batch.
|
||||
if PORT="$PORT" "${REPO_ROOT}/runner.sh" "$SERVER" "$SCENARIO" "$RUN_ID" \
|
||||
> "${BATCH_DIR}/${RUN_ID}.stdout.log" 2>&1; then
|
||||
# Move the runner's output dir into the batch dir.
|
||||
if [[ -d "${REPO_ROOT}/results/${RUN_ID}" ]]; then
|
||||
rm -rf "$RUN_DIR"
|
||||
mv "${REPO_ROOT}/results/${RUN_ID}" "$RUN_DIR"
|
||||
fi
|
||||
# Move the stdout log next to the run.
|
||||
mv "${BATCH_DIR}/${RUN_ID}.stdout.log" "${RUN_DIR}/run_all.log" 2>/dev/null || true
|
||||
|
||||
# Pretty-print the headline line from the result.
|
||||
if [[ -f "${RUN_DIR}/result.json" ]]; then
|
||||
rps=$(awk -F'[:,]' '/"sustained_rps"/ {gsub(/[ "]/, "", $2); print $2; exit}' "${RUN_DIR}/result.json")
|
||||
p99=$(awk -F'"' '/"p99"/ {print $4; exit}' "${RUN_DIR}/result.json")
|
||||
printf ' sustained=%s rps p99=%s\n' "$rps" "$p99"
|
||||
fi
|
||||
succeeded+=("$r")
|
||||
else
|
||||
echo " FAILED (see ${BATCH_DIR}/${RUN_ID}.stdout.log)"
|
||||
failed+=("$r")
|
||||
# Also try to salvage anything runner left behind for debugging.
|
||||
if [[ -d "${REPO_ROOT}/results/${RUN_ID}" ]]; then
|
||||
mkdir -p "${BATCH_DIR}/${RUN_ID}"
|
||||
mv "${REPO_ROOT}/results/${RUN_ID}"/* "${BATCH_DIR}/${RUN_ID}/" 2>/dev/null || true
|
||||
rmdir "${REPO_ROOT}/results/${RUN_ID}" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
END_TS=$(date +%s)
|
||||
ELAPSED=$((END_TS - START_TS))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build summary.md
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The headline-table shape from urus-bench-spec.md §6 has servers as
|
||||
# columns, scenarios as rows, two metrics per scenario (RPS sustained,
|
||||
# p99). We render whatever subset of the matrix actually ran.
|
||||
|
||||
SUMMARY="${BATCH_DIR}/summary.md"
|
||||
INDEX="${BATCH_DIR}/index.json"
|
||||
|
||||
# Discover every server / scenario actually represented in the batch.
|
||||
declare -A by_run # by_run["server scenario"] = path to result.json
|
||||
servers_seen=()
|
||||
scenarios_seen=()
|
||||
for d in "$BATCH_DIR"/*/; do
|
||||
rj="${d}result.json"
|
||||
[[ -f "$rj" ]] || continue
|
||||
srv=$(awk -F'"' '/"server":/ {print $4; exit}' "$rj")
|
||||
scn=$(awk -F'"' '/"scenario":/ {print $4; exit}' "$rj")
|
||||
[[ -z "$srv" || -z "$scn" ]] && continue
|
||||
by_run["$srv $scn"]="$rj"
|
||||
# Track distinct lists (order preserved by first occurrence).
|
||||
if ! printf '%s\n' "${servers_seen[@]}" | grep -qx "$srv"; then
|
||||
servers_seen+=("$srv")
|
||||
fi
|
||||
if ! printf '%s\n' "${scenarios_seen[@]}" | grep -qx "$scn"; then
|
||||
scenarios_seen+=("$scn")
|
||||
fi
|
||||
done
|
||||
|
||||
# Sort scenarios in canonical order s1..s4.
|
||||
IFS=$'\n' scenarios_seen=($(printf '%s\n' "${scenarios_seen[@]}" | sort))
|
||||
unset IFS
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit summary.md
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
{
|
||||
echo "# urus-bench summary"
|
||||
echo
|
||||
echo "- **Batch:** \`${BATCH}\`"
|
||||
echo "- **Host:** \`$(uname -n)\` (\`$(uname -srm)\`)"
|
||||
echo "- **CPUs:** \`$(nproc) cores\`"
|
||||
if [[ -r /proc/cpuinfo ]]; then
|
||||
model=$(awk -F: '/^model name/ {print $2; exit}' /proc/cpuinfo | sed 's/^ *//')
|
||||
[[ -n "$model" ]] && echo "- **Model:** ${model}"
|
||||
fi
|
||||
echo "- **Started:** \`$(date -d "@$START_TS" -Iseconds 2>/dev/null || date -Iseconds)\`"
|
||||
echo "- **Elapsed:** ${ELAPSED}s"
|
||||
echo "- **Successful runs:** ${#succeeded[@]} / ${total}"
|
||||
if [[ "${#failed[@]}" -gt 0 ]]; then
|
||||
echo "- **Failed runs:** ${failed[*]}"
|
||||
fi
|
||||
echo
|
||||
echo "Timings: probe=${PROBE_SEC:-30}s, warmup=${WARMUP_SEC:-30}s, measure=${MEASURE_SEC:-60}s, sat_ratio=${SAT_RATIO:-0.70}"
|
||||
echo
|
||||
|
||||
if [[ "${#servers_seen[@]}" -eq 0 ]]; then
|
||||
echo "_No successful runs to summarize._"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Sustained RPS table -----------------------------------------------
|
||||
echo "## Sustained RPS (wrk2 @ ${SAT_RATIO:-0.70} of saturation)"
|
||||
echo
|
||||
printf '| '
|
||||
for srv in "${servers_seen[@]}"; do printf '| %-14s ' "$srv"; done
|
||||
printf '|\n'
|
||||
printf '|------------'
|
||||
for _ in "${servers_seen[@]}"; do printf '|----------------'; done
|
||||
printf '|\n'
|
||||
for scn in "${scenarios_seen[@]}"; do
|
||||
printf '| %-10s ' "$scn"
|
||||
for srv in "${servers_seen[@]}"; do
|
||||
rj="${by_run["$srv $scn"]:-}"
|
||||
if [[ -n "$rj" ]]; then
|
||||
rps=$(awk -F'[:,]' '/"sustained_rps"/ {gsub(/[ "]/, "", $2); print $2; exit}' "$rj")
|
||||
# round to integer for readability
|
||||
rps=$(awk -v x="$rps" 'BEGIN {printf "%d", x+0.5}')
|
||||
printf '| %14s ' "$rps"
|
||||
else
|
||||
printf '| %14s ' "—"
|
||||
fi
|
||||
done
|
||||
printf '|\n'
|
||||
done
|
||||
echo
|
||||
|
||||
# --- p99 table ----------------------------------------------------------
|
||||
echo "## p99 latency (at target rate)"
|
||||
echo
|
||||
printf '| '
|
||||
for srv in "${servers_seen[@]}"; do printf '| %-14s ' "$srv"; done
|
||||
printf '|\n'
|
||||
printf '|------------'
|
||||
for _ in "${servers_seen[@]}"; do printf '|----------------'; done
|
||||
printf '|\n'
|
||||
for scn in "${scenarios_seen[@]}"; do
|
||||
printf '| %-10s ' "$scn"
|
||||
for srv in "${servers_seen[@]}"; do
|
||||
rj="${by_run["$srv $scn"]:-}"
|
||||
if [[ -n "$rj" ]]; then
|
||||
p99=$(awk -F'"' '/"p99"/ {print $4; exit}' "$rj")
|
||||
printf '| %14s ' "$p99"
|
||||
else
|
||||
printf '| %14s ' "—"
|
||||
fi
|
||||
done
|
||||
printf '|\n'
|
||||
done
|
||||
echo
|
||||
|
||||
# --- Spec pass/fail check, urus vs axum --------------------------------
|
||||
# Per spec §6: urus RPS >= 50% of axum RPS on the same scenario.
|
||||
printf '## Spec checks (urus vs axum)\n\n'
|
||||
printf '| Scenario | urus RPS | axum RPS | urus/axum | within 2× of axum |\n'
|
||||
printf '|----------|----------|----------|-----------|-------------------|\n'
|
||||
for scn in "${scenarios_seen[@]}"; do
|
||||
# Match urus-{mem,sqlite} against axum-{mem,sqlite} for the matching tier.
|
||||
urus_run=""; axum_run=""
|
||||
for k in "${!by_run[@]}"; do
|
||||
set -- $k
|
||||
s="$1"; sc="$2"
|
||||
[[ "$sc" != "$scn" ]] && continue
|
||||
case "$s" in
|
||||
urus-*) urus_run="${by_run[$k]}" ;;
|
||||
axum-*) axum_run="${by_run[$k]}" ;;
|
||||
esac
|
||||
done
|
||||
if [[ -n "$urus_run" && -n "$axum_run" ]]; then
|
||||
u=$(awk -F'[:,]' '/"sustained_rps"/ {gsub(/[ "]/, "", $2); print $2; exit}' "$urus_run")
|
||||
a=$(awk -F'[:,]' '/"sustained_rps"/ {gsub(/[ "]/, "", $2); print $2; exit}' "$axum_run")
|
||||
ratio=$(awk -v u="$u" -v a="$a" 'BEGIN { if (a+0 > 0) printf "%.2f", (u+0)/(a+0); else print "n/a" }')
|
||||
pass=$(awk -v r="$ratio" 'BEGIN { print (r != "n/a" && r+0 >= 0.5) ? "✅" : "❌" }')
|
||||
printf '| %-8s | %8.0f | %8.0f | %9s | %-17s |\n' "$scn" "$u" "$a" "$ratio" "$pass"
|
||||
else
|
||||
printf '| %-8s | %8s | %8s | %9s | %-17s |\n' "$scn" "—" "—" "—" "—"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
# --- per-run detail links ----------------------------------------------
|
||||
echo "## Per-run artefacts"
|
||||
echo
|
||||
for r in "${succeeded[@]}"; do
|
||||
set -- $r
|
||||
s="$1"; sc="$2"
|
||||
echo "- \`${s}-${sc}/\` — [result.json](${s}-${sc}/result.json), [measure.txt](${s}-${sc}/measure.txt), [server.log](${s}-${sc}/server.log)"
|
||||
done
|
||||
if [[ "${#failed[@]}" -gt 0 ]]; then
|
||||
echo
|
||||
echo "### Failed runs"
|
||||
for r in "${failed[@]}"; do
|
||||
set -- $r
|
||||
s="$1"; sc="$2"
|
||||
echo "- \`${s}-${sc}\` — see \`${s}-${sc}.stdout.log\`"
|
||||
done
|
||||
fi
|
||||
} > "$SUMMARY"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emit machine-readable index.json
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
{
|
||||
echo "{"
|
||||
echo " \"batch\": \"${BATCH}\","
|
||||
echo " \"elapsed_seconds\": ${ELAPSED},"
|
||||
echo " \"runs\": ["
|
||||
first=1
|
||||
for k in "${!by_run[@]}"; do
|
||||
set -- $k
|
||||
s="$1"; sc="$2"
|
||||
rj="${by_run[$k]}"
|
||||
[[ "$first" -eq 1 ]] || echo " ,"
|
||||
first=0
|
||||
echo " {"
|
||||
echo " \"server\": \"${s}\","
|
||||
echo " \"scenario\": \"${sc}\","
|
||||
echo " \"result_path\": \"${s}-${sc}/result.json\","
|
||||
# Inline the result.json contents, indented; simplest is to just paste.
|
||||
awk '/^\{/{p=1} p{print " " $0} /^\}/{p=0}' "$rj" \
|
||||
| sed '1d' | head -n -1 | sed 's/^/ "result": {/; t; s/^/ /' >/dev/null
|
||||
# Simpler: just reference the file path. The result.json is already
|
||||
# right there. We don't need to inline.
|
||||
echo " }"
|
||||
done
|
||||
echo " ]"
|
||||
echo "}"
|
||||
} > "$INDEX"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wrap up
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
echo
|
||||
echo "=========================================================="
|
||||
echo "run_all: DONE"
|
||||
echo " batch: ${BATCH}"
|
||||
echo " elapsed: ${ELAPSED}s"
|
||||
echo " successful: ${#succeeded[@]} / ${total}"
|
||||
[[ "${#failed[@]}" -gt 0 ]] && echo " failed: ${failed[*]}"
|
||||
echo " summary: ${SUMMARY}"
|
||||
echo "=========================================================="
|
||||
echo
|
||||
cat "$SUMMARY"
|
||||
Reference in New Issue
Block a user