sandbox-agent
926aba4344
The first multi-core A/B was void: ~100% non-2xx on every row, so 'sustained rps' was error-path throughput, not store work -- the spin patch's park/wake path was never exercised. The harness reported it cheerfully anyway. - spin_ab_urus.sh: per-run output now computes non-2xx vs approx total and prints 'VOID: N% non-2xx' instead of a meaningless rps when >2% error. - loadgen/_diag_status.lua: single-thread wrk script that tallies exact status codes (response() + done()) and names the likely cause (401=auth not reaching server / 404=store not pre-populated / 503=store actor down). The aggregate 'Non-2xx' count can't distinguish these; this can. Root cause still open: locally the store survives heavy keep-alive load (no panic), prepop populates, auth+happy-path serve 200s -- so the storm isn't reproducible in this 1-core sandbox. s3 at ~100% (not ~80%) rules out a plain 404 storm and points at auth or store-call erroring on every request; needs the diag run on the real box to confirm.
68 lines
2.6 KiB
Lua
68 lines
2.6 KiB
Lua
-- _diag_status.lua — diagnostic: tally response status codes and print them.
|
|
--
|
|
-- The normal bench scripts report only an aggregate "Non-2xx" count, which
|
|
-- can't tell a 401 (auth not reaching the server) from a 404 (store not
|
|
-- populated) from a 503 (store-call erroring). This script counts every
|
|
-- status code and prints the breakdown in done().
|
|
--
|
|
-- IMPORTANT: run with a SINGLE wrk thread (-t1) so response() and done()
|
|
-- share one Lua VM and the counter aggregates. Low load is fine; this is a
|
|
-- correctness probe, not a throughput test.
|
|
--
|
|
-- Usage (against an already-running, pre-populated server):
|
|
-- HOST=127.0.0.1:8080 BEARER=test-token-aaaaaaaaaaaaaaaaaaaa \
|
|
-- wrk -t1 -c10 -d3s -s loadgen/_diag_status.lua http://127.0.0.1:8080
|
|
--
|
|
-- It sends the S3 mix (GET-one / list / POST) so it exercises auth + store
|
|
-- on every route. If you see mostly 401 -> the auth header isn't reaching the
|
|
-- server (lua/header issue). Mostly 404 -> the store wasn't pre-populated.
|
|
-- Mostly 503 -> store-call is erroring (store actor down). Mostly 2xx at low
|
|
-- load but the full bench shows non-2xx -> it's load/scale dependent.
|
|
|
|
local common = require("common")
|
|
wrk.headers = common.headers
|
|
wrk.headers["Content-Type"] = "application/json"
|
|
|
|
math.randomseed(os.time() + os.getpid())
|
|
|
|
local body = [[{"name":"diag","email":"diag@example.test"}]]
|
|
|
|
request = function()
|
|
local r = math.random()
|
|
if r < 0.80 then
|
|
return wrk.format("GET", "/api/v1/users/" .. math.random(1, 10000))
|
|
elseif r < 0.95 then
|
|
return wrk.format("GET", "/api/v1/users")
|
|
else
|
|
return wrk.format("POST", "/api/v1/users", nil, body)
|
|
end
|
|
end
|
|
|
|
local codes = {}
|
|
local total = 0
|
|
|
|
response = function(status, headers, body)
|
|
codes[status] = (codes[status] or 0) + 1
|
|
total = total + 1
|
|
end
|
|
|
|
done = function(summary, latency, requests)
|
|
io.write("\n=== status code distribution (single-thread tally) ===\n")
|
|
-- sort keys for stable output
|
|
local keys = {}
|
|
for k in pairs(codes) do keys[#keys + 1] = k end
|
|
table.sort(keys)
|
|
for _, k in ipairs(keys) do
|
|
local n = codes[k]
|
|
io.write(string.format(" %d : %d (%.1f%%)\n", k, n, 100.0 * n / total))
|
|
end
|
|
io.write(string.format(" total tallied: %d\n", total))
|
|
if codes[401] and codes[401] > total * 0.5 then
|
|
io.write(" -> mostly 401: the auth header is NOT reaching the server.\n")
|
|
elseif codes[404] and codes[404] > total * 0.5 then
|
|
io.write(" -> mostly 404: the store was NOT pre-populated (or id range mismatch).\n")
|
|
elseif codes[503] and codes[503] > total * 0.5 then
|
|
io.write(" -> mostly 503: store-call is erroring (store actor down).\n")
|
|
end
|
|
end
|