feat(serve): graceful shutdown + connection registry (v0.2 chunk 2)
Shutdown is drain-then-force-stop: flag.store(true) -> sup_h.join() (the no-more-accepts barrier) -> cast BeginDrain -> poll ConnCount every 50ms until 0 or drain_timeout; past the deadline each tick casts ForceStopConns and re-sweeps, so conns that registered after the deadline are still caught. Listener shutdown is redesigned vs chunk 1: Restart::Permanent -> Transient, wait-failure return -> panic (Transient restarts it), and a shared Arc<AtomicBool> flag checked at loop top with a 250ms wait_readable_timeout tick instead of request_stop — no request_stop on the supervisor or listeners anywhere. Historical note: the redesign was originally forced by smarm's then-lossy stop against QUEUED actors (fixed upstream in 7bab4d2); it is kept because flag-based listener shutdown is simpler and stop-semantics-free. Conns self-register with the registry (ConnStarted from the conn actor, not a listener cast): per-sender FIFO ordering is the only thing that prevents ConnEnded overtaking ConnStarted; see conn_registry module docs. conn_actor: in the catch_unwind(pipeline.run) Err branch, smarm::preempt::check_cancelled() runs BEFORE composing the 500 — smarm's stop is an undowncastable panic_any(StopSentinel), so the catch_unwind would otherwise swallow a stop and 500-and-keep-running. The stop flag is persistent, so check_cancelled re-raises cleanly. Handle.shutdown() wakes the root via a 100ms try_recv poll (SHUTDOWN_POLL): a cross-thread Sender::send's unpark is a try_with_runtime no-op without runtime TLS on the sending thread (still true on smarm 8e5b754; cross-thread unpark is a recorded smarm roadmap candidate — this poll dies with it). Two smarm bugs were found during this chunk and fixed upstream: lossy stop against QUEUED actors (7bab4d2) and the terminal-wake shutdown stall/hang (eddf3fe); post-mortem in artefact smarm-bug-terminal-wake.md.
This commit is contained in:
Claude
@@ -284,3 +284,144 @@ fn panicking_listener_restarts() {
|
||||
assert_eq!(http_status(&resp), 200);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Graceful shutdown (v0.2 chunk 2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Boot a server with a shutdown handle. Returns (port, handle, done_rx)
|
||||
/// where done_rx fires when serve_with_shutdown returns.
|
||||
fn spawn_server_with_handle(
|
||||
pipeline: Pipeline,
|
||||
drain: Duration,
|
||||
) -> (u16, urus::Handle, std::sync::mpsc::Receiver<()>) {
|
||||
let port = free_port();
|
||||
let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
|
||||
let (handle, signal) = urus::shutdown_handle();
|
||||
let (done_tx, done_rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let cfg = Config {
|
||||
listener_pool: 2,
|
||||
scheduler_threads: Some(2),
|
||||
drain_timeout: drain,
|
||||
..Config::new(addr)
|
||||
};
|
||||
urus::serve_with_shutdown(cfg, pipeline, signal).unwrap();
|
||||
let _ = done_tx.send(());
|
||||
});
|
||||
for _ in 0..50 {
|
||||
if TcpStream::connect(addr).is_ok() {
|
||||
return (port, handle, done_rx);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
panic!("server didn't come up on {addr}");
|
||||
}
|
||||
|
||||
/// Shutdown with nothing in flight returns promptly — well inside the
|
||||
/// (deliberately huge) drain window, proving idle conns don't run the
|
||||
/// clock out.
|
||||
#[test]
|
||||
fn shutdown_with_no_connections_returns() {
|
||||
let pipe = Pipeline::new().plug(
|
||||
Router::new().get("/", |c: Conn, _n: Next| c.put_status(200))
|
||||
);
|
||||
let (_port, handle, done_rx) = spawn_server_with_handle(pipe, Duration::from_secs(30));
|
||||
handle.shutdown();
|
||||
done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("serve_with_shutdown did not return after shutdown()");
|
||||
}
|
||||
|
||||
/// An idle keep-alive connection is closed immediately on shutdown (the
|
||||
/// drain deadline of 30s must NOT be what gates the return), and the
|
||||
/// server refuses new connections afterwards.
|
||||
#[test]
|
||||
fn shutdown_closes_idle_keepalive_promptly() {
|
||||
let pipe = Pipeline::new().plug(
|
||||
Router::new().get("/", |c: Conn, _n: Next| c.put_status(200).put_body("ok"))
|
||||
);
|
||||
let (port, handle, done_rx) = spawn_server_with_handle(pipe, Duration::from_secs(30));
|
||||
|
||||
// Complete one request on a keep-alive connection and leave it open.
|
||||
let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
||||
s.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
|
||||
s.write_all(b"GET / HTTP/1.1\r\nHost: x\r\n\r\n").unwrap();
|
||||
let mut buf = [0u8; 1024];
|
||||
let n = s.read(&mut buf).unwrap();
|
||||
assert!(n > 0 && buf.starts_with(b"HTTP/1.1 200"));
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
handle.shutdown();
|
||||
|
||||
// The parked conn actor is stopped; its socket closes → we see EOF.
|
||||
let n = s.read(&mut buf).expect("read after shutdown");
|
||||
assert_eq!(n, 0, "expected EOF on the idle keep-alive connection");
|
||||
|
||||
done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("serve did not return");
|
||||
assert!(
|
||||
t0.elapsed() < Duration::from_secs(5),
|
||||
"shutdown waited for the drain deadline instead of stopping the idle conn"
|
||||
);
|
||||
|
||||
// Listeners are gone: new connections are refused.
|
||||
assert!(
|
||||
TcpStream::connect(("127.0.0.1", port)).is_err(),
|
||||
"server still accepting after shutdown"
|
||||
);
|
||||
}
|
||||
|
||||
/// A request in flight when shutdown is signalled completes with a real
|
||||
/// response before its connection is closed.
|
||||
#[test]
|
||||
fn shutdown_drains_in_flight_request() {
|
||||
let pipe = Pipeline::new().plug(
|
||||
Router::new().get("/slow", |c: Conn, _n: Next| {
|
||||
smarm::sleep(Duration::from_millis(400));
|
||||
c.put_status(200).put_body("made it")
|
||||
})
|
||||
);
|
||||
let (port, handle, done_rx) = spawn_server_with_handle(pipe, Duration::from_secs(10));
|
||||
|
||||
// Fire the slow request, then shut down while it is in flight.
|
||||
let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
||||
s.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
|
||||
s.write_all(b"GET /slow HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").unwrap();
|
||||
std::thread::sleep(Duration::from_millis(100)); // let the head land
|
||||
handle.shutdown();
|
||||
|
||||
let mut resp = Vec::new();
|
||||
s.read_to_end(&mut resp).unwrap();
|
||||
assert_eq!(http_status(&resp), 200, "in-flight request was cut off by shutdown");
|
||||
assert_eq!(http_body(&resp), b"made it");
|
||||
|
||||
done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("serve did not return after drain");
|
||||
}
|
||||
|
||||
/// A request still in flight at the drain deadline is force-stopped: the
|
||||
/// connection dies without a response, but serve returns near the
|
||||
/// deadline rather than hanging on the stuck handler's full duration.
|
||||
#[test]
|
||||
fn shutdown_force_stops_at_drain_deadline() {
|
||||
let pipe = Pipeline::new().plug(
|
||||
Router::new().get("/stuck", |c: Conn, _n: Next| {
|
||||
smarm::sleep(Duration::from_secs(60));
|
||||
c.put_status(200)
|
||||
})
|
||||
);
|
||||
let (port, handle, done_rx) = spawn_server_with_handle(pipe, Duration::from_millis(300));
|
||||
|
||||
let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
||||
s.set_read_timeout(Some(Duration::from_secs(10))).unwrap();
|
||||
s.write_all(b"GET /stuck HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").unwrap();
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
handle.shutdown();
|
||||
|
||||
done_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("serve did not return after force-stop deadline");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user