From 926aba4344d6a3e8f680daec1c0283dafe8cea09 Mon Sep 17 00:00:00 2001
From: sandbox-agent <agent@sandbox.local>
Date: Sun, 14 Jun 2026 21:58:27 +0000
Subject: [PATCH] bench: flag void (high non-2xx) runs; add status-code
 diagnostic

The first multi-core A/B was void: ~100% non-2xx on every row, so 'sustained
rps' was error-path throughput, not store work -- the spin patch's park/wake
path was never exercised. The harness reported it cheerfully anyway.

- spin_ab_urus.sh: per-run output now computes non-2xx vs approx total and
  prints 'VOID: N% non-2xx' instead of a meaningless rps when >2% error.
- loadgen/_diag_status.lua: single-thread wrk script that tallies exact status
  codes (response() + done()) and names the likely cause (401=auth not
  reaching server / 404=store not pre-populated / 503=store actor down).
  The aggregate 'Non-2xx' count can't distinguish these; this can.

Root cause still open: locally the store survives heavy keep-alive load (no
panic), prepop populates, auth+happy-path serve 200s -- so the storm isn't
reproducible in this 1-core sandbox. s3 at ~100% (not ~80%) rules out a plain
404 storm and points at auth or store-call erroring on every request; needs
the diag run on the real box to confirm.
---
 loadgen/_diag_status.lua | 67 ++++++++++++++++++++++++++++++++++++++++
 spin_ab_urus.sh          | 13 ++++++--
 2 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 loadgen/_diag_status.lua

diff --git a/loadgen/_diag_status.lua b/loadgen/_diag_status.lua
new file mode 100644
index 0000000..a3c7cf9
--- /dev/null
+++ b/loadgen/_diag_status.lua
@@ -0,0 +1,67 @@
+-- _diag_status.lua — diagnostic: tally response status codes and print them.
+--
+-- The normal bench scripts report only an aggregate "Non-2xx" count, which
+-- can't tell a 401 (auth not reaching the server) from a 404 (store not
+-- populated) from a 503 (store-call erroring). This script counts every
+-- status code and prints the breakdown in done().
+--
+-- IMPORTANT: run with a SINGLE wrk thread (-t1) so response() and done()
+-- share one Lua VM and the counter aggregates. Low load is fine; this is a
+-- correctness probe, not a throughput test.
+--
+-- Usage (against an already-running, pre-populated server):
+--   HOST=127.0.0.1:8080 BEARER=test-token-aaaaaaaaaaaaaaaaaaaa \
+--     wrk -t1 -c10 -d3s -s loadgen/_diag_status.lua http://127.0.0.1:8080
+--
+-- It sends the S3 mix (GET-one / list / POST) so it exercises auth + store
+-- on every route. If you see mostly 401 -> the auth header isn't reaching the
+-- server (lua/header issue). Mostly 404 -> the store wasn't pre-populated.
+-- Mostly 503 -> store-call is erroring (store actor down). Mostly 2xx at low
+-- load but the full bench shows non-2xx -> it's load/scale dependent.
+
+local common = require("common")
+wrk.headers = common.headers
+wrk.headers["Content-Type"] = "application/json"
+
+math.randomseed(os.time() + os.getpid())
+
+local body = [[{"name":"diag","email":"diag@example.test"}]]
+
+request = function()
+  local r = math.random()
+  if r < 0.80 then
+    return wrk.format("GET", "/api/v1/users/" .. math.random(1, 10000))
+  elseif r < 0.95 then
+    return wrk.format("GET", "/api/v1/users")
+  else
+    return wrk.format("POST", "/api/v1/users", nil, body)
+  end
+end
+
+local codes = {}
+local total = 0
+
+response = function(status, headers, body)
+  codes[status] = (codes[status] or 0) + 1
+  total = total + 1
+end
+
+done = function(summary, latency, requests)
+  io.write("\n=== status code distribution (single-thread tally) ===\n")
+  -- sort keys for stable output
+  local keys = {}
+  for k in pairs(codes) do keys[#keys + 1] = k end
+  table.sort(keys)
+  for _, k in ipairs(keys) do
+    local n = codes[k]
+    io.write(string.format("  %d : %d  (%.1f%%)\n", k, n, 100.0 * n / total))
+  end
+  io.write(string.format("  total tallied: %d\n", total))
+  if codes[401] and codes[401] > total * 0.5 then
+    io.write("  -> mostly 401: the auth header is NOT reaching the server.\n")
+  elseif codes[404] and codes[404] > total * 0.5 then
+    io.write("  -> mostly 404: the store was NOT pre-populated (or id range mismatch).\n")
+  elseif codes[503] and codes[503] > total * 0.5 then
+    io.write("  -> mostly 503: store-call is erroring (store actor down).\n")
+  end
+end
diff --git a/spin_ab_urus.sh b/spin_ab_urus.sh
index 61cdd74..7e83f5d 100755
--- a/spin_ab_urus.sh
+++ b/spin_ab_urus.sh
@@ -200,10 +200,19 @@ run_commit() {
             mv "${BENCH_DIR}/results/${run_id}/"* "$dest/" 2>/dev/null || true
             rmdir "${BENCH_DIR}/results/${run_id}" 2>/dev/null || true
           fi
-          local rps p99
+          local rps p99 non2xx ms errpct
           rps=$(awk -F'[:,]' '/"sustained_rps"/ {gsub(/[ "]/,"",$2); print $2; exit}' "${dest}/result.json" 2>/dev/null || echo "?")
           p99=$(awk -F'"' '/"p99"/ {print $4; exit}' "${dest}/result.json" 2>/dev/null || echo "?")
-          echo "      -> sustained=${rps} rps  p99=${p99}"
+          non2xx=$(awk -F'[:,]' '/"non_2xx"/ {gsub(/[ "]/,"",$2); print $2; exit}' "${dest}/result.json" 2>/dev/null || echo 0)
+          ms="${MEASURE_SEC:-60}"
+          # error rate vs approx total requests (sustained_rps * measure_sec).
+          errpct=$(awk -v n="$non2xx" -v r="$rps" -v s="$ms" \
+                     'BEGIN { t=r*s; if (t>0) printf "%.1f", 100*n/t; else print "?" }')
+          if awk -v e="$errpct" 'BEGIN { exit !(e+0 > 2.0) }' 2>/dev/null; then
+            echo "      -> VOID: ${errpct}% non-2xx (${non2xx}) — server is erroring, not serving. rps=${rps} is meaningless."
+          else
+            echo "      -> sustained=${rps} rps  p99=${p99}  (non-2xx ${errpct}%)"
+          fi
         else
           echo "      -> FAILED (see ${dest}/runner.log)"
         fi