feat(cluster): RFC 010 c10–c16, follow-ups and Phase 6 (squash of 16ef583..d9c62a8)
Tree snapshot of d9c62a8 (2026-08-18). The 20 source commits between
16ef583 (c9) and d9c62a8 were never pushed and the clone that held them
was lost; this commit carries their combined tree verbatim so the build
history stays auditable from the c1–c9 commits below it. Original
hashes as recorded in the session handoff:
c10 f03e94d pid targeting + auto-serialization (RemotePid, D14 name
on the wire); Phase 3 gate
c11 7ef4bad DownReason::Disconnected, wire tag 5
c12 d124162 remote monitors (Monitor/Demonitor/Down frames)
c13 9de967b connection-loss synthesis (A+B: Monitors::teardown +
unread-command Disconnected); Phase 4 gate
c14 7e822b7 eager pg eviction (reaper actor, ReaperInboxes)
dbe1a22 InboundVerdict::label(), trace::Event::ClusterInbound
31a9877 tests/channel.rs monitor-churn target gated on `go`
653559e Discovery::Withdrawn{name, addr}
c15 b41d76e distributed pg: Sync on NodeUp, Join/Leave broadcast,
NodeDown sweep, members_all; PgMsg wire type
c16 fafa881 pick_any / dispatch_any; Phase 5 complete
Phase 6 Tier A:
195c73e p4 NodeEvent::NodeDown(NodeInfo)
48fd766 p1 connector Candidate{name, addr, state}
ce8cf99 p2+p7 conn.rs select arms as Vec<Arm>; Outbound::Drained
bf24988 p6 RemotePid::from_local -> Option
9ae0380 p3 PeerStanding{Free, Claimed, Dialing}
c7d62a1 p11 cluster::Timing knobs, threaded by value
46f171d p11 cluster_disconnect un-ignored on SMARM_FAST_TIMING
Phase 6 Tier B:
391a9ae p5 cluster::RemoteDownReason{Local, Disconnected};
DownReason::Disconnected removed from core
7ddd908 p9 pg ctl channel unconditional, one cfg seam at spawn
d9c62a8 PeerNameMismatch parks the candidate; ClusterDial trace
Verified at d9c62a8: default 361/0, cluster 448/0, clippy --lib on
default / cluster / cluster+smarm-trace, fmt, 10x flake on
cluster_dial_mismatch, 5x on cluster_pg.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
//! RFC 010 — `Discovery::Withdrawn`: a strategy retracts a candidate and the
|
||||
//! connector stops dialing it.
|
||||
//!
|
||||
//! Cross-process: a plain *server* node, and a *client* whose strategy is a
|
||||
//! script: announce a decoy `(ghost, addr)` where `addr` is a raw
|
||||
//! `TcpListener` the client itself holds (an OS thread accepts and
|
||||
//! immediately closes, so every dial fails at handshake and the connector
|
||||
//! keeps retrying on backoff — the accept count is the dial count); after a
|
||||
//! beat, withdraw the decoy and announce the real server. The client waits
|
||||
//! for the server's `node_up` — which is *after* the withdrawal in the
|
||||
//! strategy's own stream — then watches the decoy's accept count stay flat
|
||||
//! across a window longer than the pending backoff. Before withdrawal it
|
||||
//! must have been climbing (≥ 1), or the negative proves nothing.
|
||||
#![cfg(feature = "cluster")]
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{maybe_child, spawn_node};
|
||||
use smarm::channel::Sender;
|
||||
use smarm::cluster::discovery::{Discovery, Strategy};
|
||||
use smarm::cluster::envelope::NodeMeta;
|
||||
use smarm::cluster::membership::{subscribe, NodeEvent};
|
||||
use smarm::cluster::{start, Config, StaticSeeds, Timing};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ROLES: &[(&str, fn())] = &[("server", role_server), ("client", role_client)];
|
||||
|
||||
fn meta() -> NodeMeta {
|
||||
NodeMeta {
|
||||
role: "withdraw".into(),
|
||||
region: "local".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn role_server() {
|
||||
smarm::run(|| {
|
||||
let cluster = start(Config {
|
||||
node_name: "server".into(),
|
||||
meta: meta(),
|
||||
listen_addr: std::env::var("SMARM_LISTEN_ADDR")
|
||||
.unwrap_or_else(|_| "127.0.0.1:0".into()),
|
||||
strategy: Box::new(StaticSeeds::new(Vec::<(String, String)>::new())),
|
||||
timing: Timing::default(),
|
||||
})
|
||||
.expect("binds");
|
||||
println!("LISTENING {}", cluster.local_addr());
|
||||
loop {
|
||||
smarm::sleep(Duration::from_secs(3600));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Scripted strategy: decoy, pause, withdraw decoy, real server, done.
|
||||
struct Script {
|
||||
decoy: String,
|
||||
server: String,
|
||||
}
|
||||
|
||||
impl Strategy for Script {
|
||||
fn run(self: Box<Self>, out: Sender<Discovery>) {
|
||||
let _ = out.send(Discovery::Candidate {
|
||||
name: "ghost".into(),
|
||||
addr: self.decoy.clone(),
|
||||
});
|
||||
// Long enough for the 250ms/500ms retries to land: ≥ 3 dials.
|
||||
smarm::sleep(Duration::from_millis(1100));
|
||||
let _ = out.send(Discovery::Withdrawn {
|
||||
name: "ghost".into(),
|
||||
addr: self.decoy,
|
||||
});
|
||||
let _ = out.send(Discovery::Candidate {
|
||||
name: "server".into(),
|
||||
addr: self.server,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn role_client() {
|
||||
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
|
||||
// The decoy: accept-and-close on an OS thread; count every accept.
|
||||
let decoy = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let decoy_addr = decoy.local_addr().unwrap().to_string();
|
||||
let dials = Arc::new(AtomicUsize::new(0));
|
||||
let counter = dials.clone();
|
||||
std::thread::spawn(move || {
|
||||
for conn in decoy.incoming() {
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
drop(conn);
|
||||
}
|
||||
});
|
||||
|
||||
smarm::run(move || {
|
||||
let _cluster = start(Config {
|
||||
node_name: "client".into(),
|
||||
meta: meta(),
|
||||
listen_addr: "127.0.0.1:0".into(),
|
||||
strategy: Box::new(Script {
|
||||
decoy: decoy_addr,
|
||||
server: server_addr,
|
||||
}),
|
||||
timing: Timing::default(),
|
||||
})
|
||||
.expect("binds");
|
||||
let ev = subscribe().unwrap();
|
||||
loop {
|
||||
match ev.rx.recv() {
|
||||
Ok(NodeEvent::NodeUp(i)) if i.name == "server" => break,
|
||||
Ok(_) => continue,
|
||||
Err(_) => panic!("manager gone"),
|
||||
}
|
||||
}
|
||||
// The withdrawal preceded the server candidate in the strategy's
|
||||
// stream, so it has been applied. Any dial that started before it
|
||||
// is bounded by the connect+handshake deadlines; let it drain, then
|
||||
// hold the count flat across a window longer than the pending
|
||||
// backoff would be (1s at this point, 2s next).
|
||||
let before = dials.load(Ordering::SeqCst);
|
||||
smarm::sleep(Duration::from_millis(500));
|
||||
let settled = dials.load(Ordering::SeqCst);
|
||||
let t0 = Instant::now();
|
||||
while t0.elapsed() < Duration::from_millis(3000) {
|
||||
smarm::sleep(Duration::from_millis(100));
|
||||
}
|
||||
let after = dials.load(Ordering::SeqCst);
|
||||
println!("WITHDRAWN before={before} settled={settled} after={after}");
|
||||
println!("CLIENT DONE");
|
||||
loop {
|
||||
smarm::sleep(Duration::from_secs(3600));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn withdrawn_candidate_is_no_longer_dialed() {
|
||||
maybe_child(ROLES);
|
||||
let mut server = spawn_node("server", &[]);
|
||||
let saddr = server.wait_listening();
|
||||
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
|
||||
let line = client.wait_line("WITHDRAWN", |l| l.starts_with("WITHDRAWN "));
|
||||
let mut nums = line
|
||||
.split_whitespace()
|
||||
.skip(1)
|
||||
.map(|kv| kv.split_once('=').unwrap().1.parse::<usize>().unwrap());
|
||||
let (before, settled, after) = (
|
||||
nums.next().unwrap(),
|
||||
nums.next().unwrap(),
|
||||
nums.next().unwrap(),
|
||||
);
|
||||
assert!(
|
||||
before >= 1,
|
||||
"decoy was never dialed; the negative proves nothing: {line}"
|
||||
);
|
||||
assert_eq!(
|
||||
settled, after,
|
||||
"connector kept dialing a withdrawn candidate: {line}"
|
||||
);
|
||||
client.wait_line("CLIENT DONE", |l| l == "CLIENT DONE");
|
||||
}
|
||||
Reference in New Issue
Block a user