Merge branch 'rfc010-cluster': RFC 010 clustering (c1–c16 + Phase 6) onto the v0.7.0 + perf-audit tree

Both lines branched from ca1c983 (v0.6.1 + try_spawn). master carried
graceful shutdown, gen_server/gen_statem lifetime, v0.7.0 and the 16
perf-audit commits; rfc010-cluster carried clustering behind
`--features cluster`. Two resolutions beyond the automatic merge:

- tests/channel.rs: both sides deflaked the spawn-then-monitor race in
  channel_ops_interleaved_with_monitor_churn_multi_thread. Kept master's
  `spawn_monitor` (monitor registered before publish) over the cluster
  side's `go`-gated spawn; same intent, API-level fix.
- src/cluster/envelope.rs: master added `DownReason::Shutdown`, which
  made the `Frame::Down` reason codec non-exhaustive. New wire tag
  DR_SHUTDOWN = 6, encoded and decoded symmetrically. `Shutdown` never
  rides in a `Down` by contract (a target that honours the request
  exits normally); the tag exists so the codec stays total. Tags 1–5
  unchanged.

Gates on the merged tree (rustc 1.98.1): default 405/0, cluster 492/0
(0 ignored beyond the 11 pre-existing `ignore` doctests), clippy --lib
-D warnings on both configs, doctests. cluster_disconnect still ~1.8s
(SMARM_FAST_TIMING plumbing intact).
This commit is contained in:
claude-asm-audit
2026-09-12 05:25:50 +00:00
45 changed files with 10384 additions and 310 deletions
+11
View File
@@ -33,6 +33,11 @@ budget-accounting = []
# and unflagged; only the optional gen_server transport sits behind this, so a # and unflagged; only the optional gen_server transport sits behind this, so a
# release build pays nothing for an observer it never starts. # release build pays nothing for an observer it never starts.
observer = [] observer = []
# RFC 010 c1: clustering. Off by default — the default build stays libc-only,
# byte-for-byte (gate checked per phase). serde is the payload contract,
# postcard the payload codec; both minimal (no default features). Everything
# cluster-shaped lives behind this flag.
cluster = ["dep:serde", "dep:postcard"]
# Run-queue selection: exactly one, compile-time (see src/run_queue.rs). # Run-queue selection: exactly one, compile-time (see src/run_queue.rs).
# Non-default variants need --no-default-features (features are additive). # Non-default variants need --no-default-features (features are additive).
rq-mutex = [] rq-mutex = []
@@ -44,12 +49,18 @@ cc = "1"
[dependencies] [dependencies]
libc = "0.2" libc = "0.2"
# RFC 010 §2 — only compiled under `--features cluster`.
serde = { version = "1", default-features = false, optional = true }
# `alloc` (not `std`): the seam serializes to Vec; postcard stays no_std-aligned.
postcard = { version = "1", default-features = false, features = ["alloc"], optional = true }
[target.'cfg(loom)'.dependencies] [target.'cfg(loom)'.dependencies]
loom = "0.7" loom = "0.7"
[dev-dependencies] [dev-dependencies]
libc = "0.2" libc = "0.2"
# derive + std for cluster envelope tests only; the lib itself never needs them
serde = { version = "1", features = ["derive"] }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "sync", "time"] } tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "sync", "time"] }
[profile.dev] [profile.dev]
+25
View File
@@ -8,4 +8,29 @@ fn main() {
.flag_if_supported("-fno-stack-clash-protection") .flag_if_supported("-fno-stack-clash-protection")
.compile("smarm_canary"); .compile("smarm_canary");
println!("cargo:rerun-if-changed=canary/canary.c"); println!("cargo:rerun-if-changed=canary/canary.c");
// RFC 010 c6d — build_hash inputs. The compile-time facts a peer must
// share for a mesh link: the exact toolchain and the declared (enabled)
// feature set. Emitted as a plain string; the hashing (FNV-1a folded
// with PROTO_VERSION) happens in src/cluster.rs where the protocol
// version actually lives — parsing it out of a source file here would
// be a second, fragile copy. Always emitted, even for non-cluster
// builds: one env var costs the default build nothing.
let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string());
let version = std::process::Command::new(&rustc)
.arg("-V")
.output()
.ok()
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.filter(|v| !v.is_empty())
.unwrap_or_else(|| "rustc-unknown".to_string());
let mut feats: Vec<String> = std::env::vars()
.filter_map(|(k, _)| k.strip_prefix("CARGO_FEATURE_").map(str::to_string))
.collect();
feats.sort();
println!(
"cargo:rustc-env=SMARM_BUILD_HASH_INPUTS={version};features={}",
feats.join(",")
);
println!("cargo:rerun-if-env-changed=RUSTC");
} }
+10
View File
@@ -263,6 +263,16 @@ impl<T> Sender<T> {
self.inner.lock().queue.len() self.inner.lock().queue.len()
} }
/// Whether the [`Receiver`] is still alive (a send would be accepted).
pub(crate) fn receiver_alive(&self) -> bool {
self.inner.lock().receiver_alive
}
/// Whether `other` is a sender of this very channel (a clone).
pub(crate) fn same_channel(&self, other: &Sender<T>) -> bool {
Arc::ptr_eq(&self.inner, &other.inner)
}
/// Push `value` onto the channel. Succeeds unconditionally as long as /// Push `value` onto the channel. Succeeds unconditionally as long as
/// the [`Receiver`] is still alive: the queue has no capacity limit, so /// the [`Receiver`] is still alive: the queue has no capacity limit, so
/// this never blocks and never fails except when the channel is closed, /// this never blocks and never fails except when the channel is closed,
+282
View File
@@ -0,0 +1,282 @@
//! RFC 010 — clustering (smarm⇄smarm, explicit remote boundary).
//!
//! c1: feature flag + optional deps. c2: the owned envelope. c3: the
//! transport trait (control connection), framed codec, and the TCP +
//! loopback impls. c5: the handshake state machine. c6: the connection
//! [`manager`] (registry) and per-peer connection actors ([`conn`]), started
//! as an explicit supervision subtree, plus the handshake on the
//! accept/connect path ([`connect`]). Everything above them lands in later
//! chunks.
pub mod conn;
pub mod connect;
pub mod connector;
pub mod discovery;
pub mod envelope;
pub mod expose;
pub mod handshake;
pub mod manager;
pub mod membership;
pub mod pg;
pub mod remote;
pub mod transport;
use std::io;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use crate::gen_server::{self, GenServerBuilder};
use crate::monitor::monitor;
use crate::pg::Incarnation;
use crate::scheduler::{sleep, spawn, JoinHandle};
use crate::supervisor::{ChildSpec, OneForOne, Restart};
use envelope::NodeMeta;
use handshake::Local;
use transport::tcp::TcpTransport;
use transport::Transport;
pub use conn::{spawn_established, ConnHandle};
pub use connect::{dial, spawn_acceptor, AcceptorHandle};
pub use connector::{spawn_connector, ConnectorHandle};
pub use discovery::{Discovery, StaticSeeds, Strategy};
pub use envelope::RemoteDownReason;
pub use expose::{expose, expose_type, type_hash, DeliverError};
pub use manager::{Manager, MANAGER};
pub use membership::{subscribe, view, MembershipEvents, NodeEvent, NodeInfo};
pub use pg::{dispatch_any, members_all, pick_any, DispatchAnyError, GroupMember, PgMsg, PG_NAME};
pub use remote::{
demonitor_remote, monitor_remote, send_to_remote, NotConnected, RemoteDown, RemoteMonitor,
RemoteName, RemotePid, RemoteSendError, ToRemoteError,
};
/// c6d — the derived build hash for [`handshake::LocalNode::build_hash`]:
/// two builds may mesh only when this matches, and it is a pure function of
/// the compile-time inputs that define wire compatibility today — the exact
/// toolchain (`rustc -V`), the declared feature set, and
/// [`envelope::PROTO_VERSION`]. FNV-1a 64 over the build-script string, then
/// the proto version folded byte-wise, so a proto bump moves the hash even
/// on an identical toolchain. The domain is deliberately lean and
/// tightenable later without a wire change — it is just a `u64`.
pub const BUILD_HASH: u64 = fold_u32(
fnv1a64(env!("SMARM_BUILD_HASH_INPUTS").as_bytes()),
envelope::PROTO_VERSION,
);
/// FNV-1a 64 (const so [`BUILD_HASH`] is a compile-time fact).
const fn fnv1a64(bytes: &[u8]) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
let mut i = 0;
while i < bytes.len() {
h ^= bytes[i] as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
i += 1;
}
h
}
/// Continue an FNV-1a state over a `u32`'s little-endian bytes.
const fn fold_u32(mut h: u64, v: u32) -> u64 {
let b = v.to_le_bytes();
let mut i = 0;
while i < b.len() {
h ^= b[i] as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
i += 1;
}
h
}
/// The control-plane timing knobs, all with today's fixed values as
/// defaults ([`Timing::default`]). One struct threaded explicitly to the
/// acceptor, the dial path, every connection actor and the connector — no
/// ambient state, so a test can run a fast mesh without touching globals.
/// Every node in a mesh should agree on `heartbeat_interval` <
/// `liveness_timeout`; nothing enforces it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Timing {
/// Idle-connection heartbeat pace. Default [`conn::HEARTBEAT_INTERVAL`].
pub heartbeat_interval: Duration,
/// Inbound silence that tears a connection down. Default
/// [`conn::LIVENESS_TIMEOUT`].
pub liveness_timeout: Duration,
/// Per-frame handshake deadline on the accept/dial path. Default
/// [`connect::HANDSHAKE_TIMEOUT`].
pub handshake_timeout: Duration,
/// Connector redial delay after the first failure. Default
/// [`connector::INITIAL_BACKOFF`].
pub initial_backoff: Duration,
/// Connector redial delay cap. Default [`connector::MAX_BACKOFF`].
pub max_backoff: Duration,
}
impl Default for Timing {
fn default() -> Self {
Timing {
heartbeat_interval: conn::HEARTBEAT_INTERVAL,
liveness_timeout: conn::LIVENESS_TIMEOUT,
handshake_timeout: connect::HANDSHAKE_TIMEOUT,
initial_backoff: connector::INITIAL_BACKOFF,
max_backoff: connector::MAX_BACKOFF,
}
}
}
/// How to run this node: its identity and how it finds peers.
pub struct Config {
/// This node's claimed name — the mesh-wide identity peers dial by and
/// the tie-break input. Must be unique across the mesh.
pub node_name: String,
/// Metadata offered in this node's `Hello`.
pub meta: NodeMeta,
/// The control-connection listen address (e.g. `"127.0.0.1:0"`; the
/// concrete bound address is [`Cluster::local_addr`]).
pub listen_addr: String,
/// The peer-discovery strategy — [`StaticSeeds`] until richer ones land.
pub strategy: Box<dyn Strategy>,
/// Heartbeat / liveness / handshake / backoff knobs; [`Timing::default`]
/// is the shipping configuration.
pub timing: Timing,
}
/// A running cluster node: the supervised [`Manager`], the acceptor over the
/// bound listener, and the connector driving its [`Strategy`]. Roles will
/// eventually mount this; until the role mechanism lands it is started by
/// hand (RFC 010 §7).
///
/// Dropping the handle stops the acceptor and connector loops (no new
/// connections in either direction) but detaches the manager subtree, which
/// — with every established connection — keeps running for the life of the
/// runtime, the same split as [`AcceptorHandle`] alone.
pub struct Cluster {
_sup: JoinHandle,
acceptor: AcceptorHandle,
connector: ConnectorHandle,
local: Local,
}
impl Cluster {
/// The concrete bound listen address, dialable as-is.
pub fn local_addr(&self) -> &str {
self.acceptor.local_addr()
}
/// This node's handshake identity (name, incarnation, build hash, meta).
pub fn local(&self) -> &Local {
&self.local
}
/// Stop accepting and dialing. Established connections stay up (they
/// belong to the manager); tear those down via the manager.
pub fn shutdown(&self) {
self.acceptor.shutdown();
self.connector.shutdown();
}
}
/// Start a cluster node: the supervised manager (blocking until it is
/// registered and ready to answer), the acceptor bound per
/// [`Config::listen_addr`], and the connector running [`Config::strategy`].
/// The node's identity is completed here: `incarnation` is
/// [`self_incarnation`] and `build_hash` is [`BUILD_HASH`] — c7 is its first
/// consumer. Errs only if the listener cannot bind.
///
/// The manager is a supervised child (restarted on crash); per-peer
/// connection actors are dynamic and monitored by the manager rather than
/// statically supervised — a lost connection is re-established by the
/// connector's dial loop, never resurrected onto a stale socket.
pub fn start(config: Config) -> io::Result<Cluster> {
let sup = spawn(|| {
OneForOne::new()
.child(ChildSpec::new(Restart::Permanent, manager_child))
.run()
});
while gen_server::whereis_server(MANAGER).is_none() {
sleep(Duration::from_millis(1));
}
let local = Local {
node_name: config.node_name,
incarnation: self_incarnation(),
build_hash: BUILD_HASH,
meta: config.meta,
};
// The wire identity serialized pids are stamped with (c10).
remote::set_local_identity(&local.node_name, local.incarnation);
// The pg actor (Phase 5): subscribes membership, owns the "pg" name.
pg::attach_cluster();
let listener = TcpTransport.listen(&config.listen_addr)?;
let acceptor = spawn_acceptor(listener, local.clone(), config.timing);
let connector = spawn_connector(
Box::new(TcpTransport),
local.clone(),
config.strategy,
config.timing,
);
Ok(Cluster {
_sup: sup,
acceptor,
connector,
local,
})
}
/// This process's incarnation epoch: milliseconds since the Unix epoch,
/// truncated to `u32`. Not a clock — its one job is separating a node from
/// its own restart (two starts of the same name land on the same value only
/// if they happen within the same millisecond modulo ~49.7 days). Seconds
/// would be too coarse: a crash-and-restart inside one second is routine
/// under supervision.
pub fn self_incarnation() -> Incarnation {
let ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0);
Incarnation::new(ms as u32)
}
/// The supervised manager child body. It *is* the child actor: it starts the
/// named manager, then parks on the manager's own termination so this actor's
/// lifetime tracks the manager's — the supervisor's restart accounting keys off
/// this actor exiting.
fn manager_child() {
let m = match GenServerBuilder::new(Manager::new()).named(MANAGER).start() {
Ok(m) => m,
// Name still held by a not-yet-reaped prior instance: return and let
// the supervisor retry under its restart policy.
Err(_) => return,
};
let _ = monitor(m.pid()).rx.recv();
}
#[cfg(test)]
mod tests {
use super::*;
/// The hash core against the published FNV-1a 64 test vectors — the
/// contract is "this is FNV-1a", not "whatever the fn does".
#[test]
fn fnv1a64_known_vectors() {
assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325);
assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c);
assert_eq!(fnv1a64(b"foobar"), 0x85944171f73967e8);
}
/// Folding the proto version continues the same FNV state: identical
/// inputs with a different version must land on a different hash.
#[test]
fn proto_version_moves_the_hash() {
let base = fnv1a64(b"same-toolchain;features=CLUSTER");
assert_ne!(fold_u32(base, 1), fold_u32(base, 2));
// And it equals hashing the bytes in one pass — the fold is a
// continuation, not a second construction.
let mut all = b"same-toolchain;features=CLUSTER".to_vec();
all.extend_from_slice(&1u32.to_le_bytes());
assert_eq!(fold_u32(base, 1), fnv1a64(&all));
}
/// The derived constant exists, is compile-time, and is not degenerate.
#[test]
fn build_hash_is_nonzero() {
const H: u64 = BUILD_HASH;
assert_ne!(H, 0);
}
}
+630
View File
@@ -0,0 +1,630 @@
//! RFC 010 c6 — the per-peer connection actor.
//!
//! One actor per established control connection. It owns the whole
//! [`FramedConn`] and, in a single [`select`](crate::select), waits on two
//! things at once: its command inbox and the connection becoming readable (the
//! [`FdArm`](crate::scheduler::FdArm) the transport hands back). That is why it
//! is a plain select-loop actor rather than a `gen_server` or `gen_statem` —
//! neither of those can fold fd-readiness into its wait, and folding it in is
//! the whole job. The single owner sends and receives on the one `FramedConn`,
//! so no read/write split is needed.
//!
//! The handshake completes *before* this actor exists (on the accept/connect
//! path — c6b) and produces the [`Peer`]; the *path* then registers the
//! connection with the [`manager`](crate::cluster::manager), which takes
//! ownership of its [`ConnHandle`] and monitors the actor, so any exit
//! deregisters the connection. The actor itself holds no authority over its
//! own lifetime: it runs until the manager drops its handle (deregistration,
//! `Disconnect`, or manager shutdown), the connection ends, or liveness
//! expires. Heartbeat send and fixed-timeout liveness are the timeout arm of
//! the same `select` (c6c): [`HEARTBEAT_INTERVAL`] paces outbound
//! [`Frame::Heartbeat`](crate::cluster::envelope::Frame::Heartbeat)s, and a
//! [`LIVENESS_TIMEOUT`] window — reset by any inbound frame — tears the
//! connection down when it empties.
//!
//! c9 adds the third arm — the connection's dedicated **outbound inbox**
//! (`Sender<Frame>` bound in the manager-maintained outbound table, D13),
//! drained onto the wire in the same loop — and inbound *interpretation*:
//! `SendNamed` goes to the one resolution seam,
//! [`remote::deliver_named`](crate::cluster::remote::deliver_named).
//! `Send` goes to the pid seam (c10). The outbound
//! sender is a separate channel from `cmd_tx` on purpose: closing it is not
//! a stop signal — lifetime authority stays with the [`ConnHandle`] (D9).
//!
//! c12 adds the monitor plane, and it lives *here* on purpose. Two tables,
//! both owned by this actor and dying with the connection:
//!
//! - **outstanding** — monitors *this* node holds on actors at the peer:
//! `monitor_id → (target, Sender<RemoteDown>)`. Fed by
//! [`MonCmd`](crate::cluster::remote::MonCmd) from `monitor_remote`; the
//! actor records the id and *then* emits the `Monitor` frame, so a `Down`
//! frame can never race an entry that isn't there yet. An inbound `Down`
//! removes the entry and delivers.
//! - **watched** — monitors the *peer* holds on actors here: `monitor_id →
//! local Monitor`. An inbound `Monitor` is admitted only for a pid that
//! was exposed or crossed the wire (`is_watchable`, D12): a corpse answers
//! with its recorded terminal reason (RFC §6), an unwatchable or unknown
//! pid with `NoProc` — indistinguishable from dead, so nothing leaks. A
//! live watchable pid gets a local monitor whose `rx` is one more arm of
//! the select; its `Down` goes back as a frame.
//!
//! Because both tables are actor state, connection loss (c13) needs no
//! second bookkeeping owner: this actor's exit is the one place that knows
//! every monitor the link was carrying. `Monitors::teardown` runs on every
//! exit path and answers each outstanding monitor with `Disconnected` —
//! the roadmap's "partition vs. death" contrast: an actor that dies sends
//! its true reason over the link, a link that dies says only that.
use std::collections::HashMap;
use std::time::{Duration, Instant};
use crate::channel::{channel, try_select_timeout, Receiver, Selectable, Sender};
use crate::cluster::envelope::{Frame, RemoteDownReason};
use crate::cluster::handshake::Peer;
use crate::cluster::manager::{Call, Registered, Reply, MANAGER};
use crate::cluster::remote::{
deliver_named, deliver_to_pid, InboundVerdict, MonCmd, RemoteDown, RemotePid,
};
use crate::cluster::transport::FramedConn;
use crate::cluster::Timing;
use crate::gen_server;
use crate::monitor::{
demonitor, is_watchable, monitor, terminal_reason, DownReason, Monitor, MonitorId,
};
use crate::pid::{Erased, Pid};
use crate::scheduler::spawn;
/// Commands to a running connection actor.
enum Cmd {
Shutdown,
}
/// The manager's authority over one connection actor: while this handle
/// lives the connection lives, and dropping it stops the actor and closes
/// the socket. Only the [`manager`](crate::cluster::manager) holds one —
/// callers of [`spawn_established`] get a [`Pid`] and no lifetime authority,
/// so a connection can never outlive, or die with, whichever actor happened
/// to establish it.
pub struct ConnHandle {
cmd_tx: Sender<Cmd>,
/// The connection's dedicated outbound inboxes — frames and monitor
/// commands. The manager moves them into the outbound table on
/// `Register` (see [`take_outbound`](ConnHandle::take_outbound)); a
/// `Duplicate` verdict drops them with the handle.
out_tx: Option<(Sender<Frame>, Sender<MonCmd>)>,
}
impl std::fmt::Debug for ConnHandle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("ConnHandle")
}
}
impl ConnHandle {
/// Ask the connection to close and exit. Idempotent, and a no-op if the
/// actor has already gone. Dropping the handle does the same thing; this
/// exists for the manager's explicit `Disconnect` path.
pub fn shutdown(&self) {
let _ = self.cmd_tx.send(Cmd::Shutdown);
}
/// Manager-only: take the outbound senders to bind into the outbound
/// table. Once, at registration.
pub(crate) fn take_outbound(&mut self) -> Option<(Sender<Frame>, Sender<MonCmd>)> {
self.out_tx.take()
}
}
/// The name was already claimed by a live connection, so this one was
/// refused; its actor has been stopped and its socket closed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RegisterRefused;
/// Spawn a connection actor for an **already-established** connection (the
/// handshake completed on the path and produced `peer`) and register it with
/// the manager, synchronously, before returning. The manager takes the
/// actor's [`ConnHandle`]; the caller gets only the [`Pid`], because
/// connection lifetime belongs to the table and not to the establishing
/// actor. A refusal has already stopped the actor and closed the socket.
pub fn spawn_established(
framed: FramedConn,
peer: Peer,
timing: Timing,
) -> Result<Pid, RegisterRefused> {
let (cmd_tx, cmd_rx) = channel();
let (out_tx, out_rx) = channel();
let (mon_tx, mon_rx) = channel();
let reg_peer = peer.clone();
let pid = spawn(move || run(framed, peer, timing, cmd_rx, out_rx, mon_rx)).pid();
match gen_server::call(
MANAGER,
Call::Register {
peer: reg_peer,
pid,
handle: ConnHandle {
cmd_tx,
out_tx: Some((out_tx, mon_tx)),
},
},
) {
Ok(Reply::Registered(Registered::Ok)) => Ok(pid),
// Duplicate name, or the manager is unreachable. Either way the
// handle went with the call and is dropped there (or never arrived
// and dropped with it), which stops the actor and closes the socket.
_ => Err(RegisterRefused),
}
}
/// Default for [`Timing::heartbeat_interval`]: how often this end emits
/// [`Frame::Heartbeat`] on an idle connection. The first one goes out
/// immediately at spawn, so the peer's liveness window starts fed.
pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(1);
/// Default for [`Timing::liveness_timeout`]: how long the connection may go
/// without a single inbound frame before it is declared dead and torn down. Any inbound frame resets the window —
/// heartbeats keep an idle connection alive, and real traffic (c8+) counts
/// for free. Fixed by design (RFC v2 §5): this is the control connection, a
/// heartbeat can never queue behind bulk traffic, so a fixed timeout is an
/// honest detector.
pub const LIVENESS_TIMEOUT: Duration = Duration::from_secs(4);
fn run(
mut framed: FramedConn,
_peer: Peer,
timing: Timing,
cmd_rx: Receiver<Cmd>,
out_rx: Receiver<Frame>,
mon_rx: Receiver<MonCmd>,
) {
let mut mons = Monitors::default();
match framed.readable_arm() {
Some(arm) => run_live(
&mut framed,
arm,
timing,
&cmd_rx,
&out_rx,
&mon_rx,
&mut mons,
),
None => run_inert(&cmd_rx),
}
framed.close();
mons.teardown(&mon_rx);
}
/// The monitor plane's two tables (module docs). Owned by the actor.
#[derive(Default)]
struct Monitors {
/// Monitors this node holds on peer actors: id → (target, delivery).
outstanding: HashMap<MonitorId, (RemotePid<Erased>, Sender<RemoteDown>)>,
/// Monitors the peer holds on local actors: id → the local monitor.
watched: HashMap<MonitorId, Monitor>,
}
impl Monitors {
/// The connection is gone, whatever the exit path (liveness expiry,
/// EOF, wire failure, commanded stop): release the peer's local
/// monitors, and answer every one of ours with `Disconnected` — nothing
/// more can be known about those actors. Commands still sitting in the
/// inbox are folded in first (a `Monitor` handed to us but never
/// processed gets its notice too; a `Demonitor` still cancels), so the
/// only registration that can miss this is one that lands after the
/// drain and before the inbox drops — the reader side backstops that
/// (`RemoteMonitor`). Entries leave the table as they are answered, and
/// this runs once per actor, so no monitor sees two notices.
fn teardown(&mut self, mon_rx: &Receiver<MonCmd>) {
for (_, m) in self.watched.drain() {
let _ = demonitor(&m);
}
while let Ok(Some(cmd)) = mon_rx.try_recv() {
match cmd {
MonCmd::Monitor { id, target, tx } => {
self.outstanding.insert(id, (target, tx));
}
MonCmd::Demonitor { id } => {
self.outstanding.remove(&id);
}
}
}
for (_, (pid, tx)) in self.outstanding.drain() {
let _ = tx.send(RemoteDown {
pid,
reason: RemoteDownReason::Disconnected,
});
}
}
/// Admit a peer's `Monitor` for local `(index, generation)`. Returns
/// the reason to answer with at once, or `None` if a live monitor was
/// installed. Corpse → recorded terminal reason (RFC §6, and only
/// watchable deaths are recorded); live watchable → monitor; anything
/// else → `NoProc`. The check-then-monitor race (dies in between) is
/// closed on the read side: a `NoProc` from a monitor we installed on a
/// live pid is upgraded through `terminal_reason` in `sweep_watched`.
fn admit(&mut self, id: MonitorId, index: u32, generation: u32) -> Option<DownReason> {
let pid = Pid::new(index, generation);
if let Some(reason) = terminal_reason(pid) {
return Some(reason);
}
if !is_watchable(pid) {
return Some(DownReason::NoProc);
}
let m = monitor(pid);
self.watched.insert(id, m);
None
}
fn cancel(&mut self, id: MonitorId) {
if let Some(m) = self.watched.remove(&id) {
let _ = demonitor(&m);
}
}
/// Collect every local `Down` that has arrived for a peer-held monitor.
fn sweep_watched(&mut self) -> Vec<(MonitorId, DownReason)> {
let mut fired = Vec::new();
for (id, m) in self.watched.iter() {
if let Ok(Some(down)) = m.rx.try_recv() {
let reason = match down.reason {
DownReason::NoProc => terminal_reason(m.target).unwrap_or(DownReason::NoProc),
r => r,
};
fired.push((*id, reason));
}
}
for (id, _) in &fired {
self.watched.remove(id);
}
fired
}
/// The peer reports a monitored actor down: deliver locally.
fn down(&mut self, id: MonitorId, reason: RemoteDownReason) {
if let Some((pid, tx)) = self.outstanding.remove(&id) {
let _ = tx.send(RemoteDown { pid, reason });
}
}
}
/// The steady-state loop over an fd-backed connection: one
/// `select_timeout` folds the command inbox, the outbound inbox, socket
/// readability, and the nearer of the two deadlines (`hb_send`,
/// `liveness`) into a single wait.
fn run_live(
framed: &mut FramedConn,
arm: crate::scheduler::FdArm,
timing: Timing,
cmd_rx: &Receiver<Cmd>,
out_rx: &Receiver<Frame>,
mon_rx: &Receiver<MonCmd>,
mons: &mut Monitors,
) {
let mut next_hb = Instant::now();
let mut live_until = Instant::now() + timing.liveness_timeout;
// The outbound senders live in the manager's table and are dropped on
// unbind; after that these arms would wake forever, so they drop out of
// the select (not a stop signal — see the module docs).
let mut out_open = true;
let mut mon_open = true;
// Which wait each select arm stands for. Built in lockstep with the
// `Selectable` vector each iteration, so a wake is decoded by name and
// never by position.
enum Arm {
Cmd,
Fd,
Out,
Mon,
/// A peer-held local monitor (any of them: firing sweeps them all).
Watched,
}
fn push<'s>(
arms: &mut Vec<&'s dyn Selectable>,
what: &mut Vec<Arm>,
s: &'s dyn Selectable,
a: Arm,
) {
arms.push(s);
what.push(a);
}
loop {
let now = Instant::now();
if now >= live_until {
break; // liveness expired: the peer is dead to us
}
if now >= next_hb {
if framed.send(&Frame::Heartbeat).is_err() {
break;
}
next_hb = now + timing.heartbeat_interval;
}
let wait = next_hb.min(live_until).saturating_duration_since(now);
let mut arms: Vec<&dyn Selectable> = Vec::new();
let mut what: Vec<Arm> = Vec::new();
push(&mut arms, &mut what, cmd_rx, Arm::Cmd);
push(&mut arms, &mut what, &arm, Arm::Fd);
if out_open {
push(&mut arms, &mut what, out_rx, Arm::Out);
}
if mon_open {
push(&mut arms, &mut what, mon_rx, Arm::Mon);
}
for m in mons.watched.values() {
push(&mut arms, &mut what, &m.rx, Arm::Watched);
}
match try_select_timeout(&arms, wait).map(|i| i.map(|i| &what[i])) {
Ok(Some(Arm::Cmd)) => {
if should_stop(cmd_rx) {
break;
}
}
Ok(Some(Arm::Fd)) => match pump_readable(framed, mons) {
Pump::Ended => break,
Pump::Frames(n) => {
if n > 0 {
live_until = Instant::now() + timing.liveness_timeout;
}
}
},
Ok(Some(Arm::Out)) => match pump_outbound(framed, out_rx) {
Outbound::Drained => {}
Outbound::Closed => out_open = false,
Outbound::WireFailed => break,
},
Ok(Some(Arm::Mon)) => match pump_moncmds(framed, mon_rx, mons) {
Outbound::Drained => {}
Outbound::Closed => mon_open = false,
Outbound::WireFailed => break,
},
Ok(Some(Arm::Watched)) => {
for (id, reason) in mons.sweep_watched() {
let frame = Frame::Down {
monitor_id: id.0,
reason: reason.into(),
};
if framed.send(&frame).is_err() {
return;
}
}
}
// A deadline passed; the top of the loop acts on whichever.
Ok(None) => {}
// The fd arm failed to register — the connection is gone.
Err(_) => break,
}
}
}
/// Drain the monitor-command inbox: record, then emit (module docs).
fn pump_moncmds(
framed: &mut FramedConn,
mon_rx: &Receiver<MonCmd>,
mons: &mut Monitors,
) -> Outbound {
loop {
match mon_rx.try_recv() {
Ok(Some(MonCmd::Monitor { id, target, tx })) => {
let frame = Frame::Monitor {
monitor_id: id.0,
index: target.index(),
generation: target.generation(),
};
mons.outstanding.insert(id, (target, tx));
if framed.send(&frame).is_err() {
return Outbound::WireFailed;
}
}
Ok(Some(MonCmd::Demonitor { id })) => {
if mons.outstanding.remove(&id).is_some()
&& framed.send(&Frame::Demonitor { monitor_id: id.0 }).is_err()
{
return Outbound::WireFailed;
}
}
Ok(None) => return Outbound::Drained,
Err(_) => return Outbound::Closed,
}
}
}
/// What one outbound-side wake (frames or monitor commands) yielded.
enum Outbound {
/// Everything queued went onto the wire; the inbox is open and empty.
Drained,
/// The manager unbound this connection's sender; nothing more will come.
Closed,
/// The socket refused a write: the connection is gone.
WireFailed,
}
/// Drain every queued outbound frame onto the wire.
fn pump_outbound(framed: &mut FramedConn, out_rx: &Receiver<Frame>) -> Outbound {
loop {
match out_rx.try_recv() {
Ok(Some(frame)) => {
if framed.send(&frame).is_err() {
return Outbound::WireFailed;
}
}
Ok(None) => return Outbound::Drained,
Err(_) => return Outbound::Closed,
}
}
}
/// No fd to select on (loopback): only a command can end the wait, and
/// neither heartbeats nor liveness run — a transport that can't report
/// readiness can't be timed either (same caveat as
/// [`FramedConn::recv_deadline`]). Loopback is a test transport; every real
/// connection is fd-backed.
fn run_inert(cmd_rx: &Receiver<Cmd>) {
loop {
let arms: [&dyn Selectable; 1] = [cmd_rx];
let _ = crate::channel::select(&arms);
if should_stop(cmd_rx) {
break;
}
}
}
/// Drain the command arm. Returns `true` when the actor should exit — a
/// shutdown was requested, or the last handle was dropped.
fn should_stop(cmd_rx: &Receiver<Cmd>) -> bool {
match cmd_rx.try_recv() {
Ok(Some(Cmd::Shutdown)) => true,
Ok(None) => false, // spurious wake
Err(_) => true, // all senders dropped
}
}
/// What one readable wake yielded.
enum Pump {
/// The connection has ended: EOF (clean or mid-frame) or an
/// unrecoverable stream error.
Ended,
/// Still up; this many complete frames were consumed (possibly zero, if
/// the wake delivered only part of a frame). Any nonzero count resets
/// the liveness window.
Frames(usize),
}
/// Surface an inbound verdict: one `smarm-trace` event, nothing else — it
/// is local knowledge (RFC §3). A no-op without the feature.
fn note_verdict(verdict: InboundVerdict) {
#[cfg(feature = "smarm-trace")]
crate::te!(crate::trace::Event::ClusterInbound(verdict.label()));
#[cfg(not(feature = "smarm-trace"))]
drop(verdict);
}
/// Consume one readable wake: exactly one socket read (which cannot block
/// after a level-triggered readable indication), then drain every complete
/// frame the buffer now holds. A blocking `recv` here would park the actor
/// past its heartbeat and liveness deadlines whenever a frame arrives split.
/// Every consumed frame counts for liveness; `SendNamed` goes to the one
/// name-resolution seam and `Send` to the pid seam. Verdicts are local
/// knowledge only — nothing goes back on the wire (RFC §3) — and surface
/// as one `smarm-trace` `ClusterInbound` event each (zero cost off).
/// `Monitor`/`Demonitor`/`Down` go to the [`Monitors`] tables; a `Monitor`
/// that can be answered at once is answered inline.
fn pump_readable(framed: &mut FramedConn, mons: &mut Monitors) -> Pump {
let eof = match framed.read_once() {
Ok(n) => n == 0,
Err(_) => return Pump::Ended,
};
let mut got = 0;
loop {
match framed.next_buffered() {
Ok(Some(frame)) => {
got += 1;
match frame {
Frame::SendNamed {
name,
type_hash,
payload,
} => {
note_verdict(deliver_named(&name, type_hash, &payload));
}
Frame::Send {
index,
generation,
type_hash,
payload,
} => {
note_verdict(deliver_to_pid(index, generation, type_hash, &payload));
}
Frame::Monitor {
monitor_id,
index,
generation,
} => {
let id = MonitorId(monitor_id);
if let Some(reason) = mons.admit(id, index, generation) {
let frame = Frame::Down {
monitor_id,
reason: reason.into(),
};
if framed.send(&frame).is_err() {
return Pump::Ended;
}
}
}
Frame::Demonitor { monitor_id } => mons.cancel(MonitorId(monitor_id)),
Frame::Down { monitor_id, reason } => mons.down(MonitorId(monitor_id), reason),
// Heartbeat: liveness only. Handshake frames after
// establishment: ignored.
_ => {}
}
}
Ok(None) => break,
Err(_) => return Pump::Ended, // corrupt stream
}
}
if eof {
Pump::Ended
} else {
Pump::Frames(got)
}
}
#[cfg(test)]
mod tests {
//! `Monitors::teardown` in isolation: the actor-side half of c13, pinned
//! separately because from the outside it is indistinguishable from the
//! read-side backstop in `RemoteMonitor` (both yield `Disconnected`).
use super::*;
use crate::pg::Incarnation;
fn pid(index: u32) -> RemotePid<Erased> {
RemotePid::from_parts("peer", Incarnation::new(1), index, 1)
}
#[test]
fn teardown_answers_every_outstanding_and_unread_monitor_once() {
crate::run(|| {
let mut mons = Monitors::default();
let (mon_tx, mon_rx) = channel::<MonCmd>();
// Already registered.
let (tx1, rx1) = channel::<RemoteDown>();
mons.outstanding.insert(MonitorId(1), (pid(1), tx1));
// In the inbox, never processed.
let (tx2, rx2) = channel::<RemoteDown>();
mon_tx
.send(MonCmd::Monitor {
id: MonitorId(2),
target: pid(2),
tx: tx2,
})
.ok()
.unwrap();
// Registered, then cancelled in the inbox: silence.
let (tx3, rx3) = channel::<RemoteDown>();
mons.outstanding.insert(MonitorId(3), (pid(3), tx3));
mon_tx
.send(MonCmd::Demonitor { id: MonitorId(3) })
.ok()
.unwrap();
mons.teardown(&mon_rx);
let d1 = rx1.recv().unwrap();
assert_eq!(
(d1.pid, d1.reason),
(pid(1), RemoteDownReason::Disconnected)
);
let d2 = rx2.recv().unwrap();
assert_eq!(
(d2.pid, d2.reason),
(pid(2), RemoteDownReason::Disconnected)
);
// Cancelled: no notice was sent (its sender is dropped, channel
// closed-empty), and nobody got a second one.
assert!(rx3.try_recv().is_err());
assert!(rx1.try_recv().is_err());
assert!(rx2.try_recv().is_err());
assert!(mons.outstanding.is_empty());
});
}
}
+365
View File
@@ -0,0 +1,365 @@
//! RFC 010 c6b — the handshake on the accept/connect path.
//!
//! Per D8 (re-amended): the c5 machines are driven by **straight-line code
//! on the path**, not by an actor. The dial side runs [`Initiator`]; the
//! acceptor loop runs [`Responder`]. A connection actor is spawned only
//! *after* a successful handshake ([`spawn_established`]); every reject,
//! protocol failure, timeout, and tie-break loss is resolved right here,
//! on the path, by closing — no actor ever exists for a connection that
//! didn't establish.
//!
//! Buffer trap (binding): the path reader and the steady-state actor share
//! ONE [`FramedConn`]. Its decode buffer may hold read-ahead past the
//! handshake frames, so the *whole* `FramedConn` travels into
//! [`spawn_established`] — never a fresh codec over the same socket.
//!
//! Layering: [`dial_handshake`] and [`accept_handshake`] are the bare path
//! steps — IO on a `FramedConn`, no manager, no actors — testable over the
//! loopback transport on plain threads. [`dial`] and [`spawn_acceptor`] are
//! the manager-integrated layer (actor context required): they keep the
//! [`manager`](crate::cluster::manager)'s dial-intent set honest and spawn
//! the connection actor on success.
use std::io;
use std::time::{Duration, Instant};
use crate::channel::{channel, Receiver, Selectable, Sender};
use crate::cluster::conn::spawn_established;
use crate::cluster::envelope::{Frame, RejectReason};
use crate::cluster::handshake::{
Initiator, InitiatorOutcome, Local, Peer, PeerStanding, Responder, ResponderOutcome,
};
use crate::cluster::manager::{Call, Reply, MANAGER};
use crate::cluster::transport::{FramedConn, Listener, RecvError, SendError, Transport};
use crate::cluster::Timing;
use crate::gen_server;
use crate::pid::Pid;
use crate::scheduler::{self, spawn};
/// Default for [`Timing::handshake_timeout`]: how long either side waits for
/// the peer's handshake frame before giving up and closing. Enforced on the path via [`FramedConn::recv_deadline`],
/// so a peer that connects and goes silent cannot wedge the acceptor.
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5);
/// Why a handshake did not establish. In every case the connection has
/// already been closed on the path by the time this is returned.
#[derive(Debug)]
pub enum HandshakeError {
/// A `HelloReject` travelled — sent by us (accept side) or received by
/// us (dial side).
Rejected(RejectReason),
/// Accept side only: the inbound dial lost the simultaneous-connect
/// tie-break (D7) and was closed silently, no frame sent.
TieBreakLoss,
/// The peer spoke a valid frame that is wrong here (non-`Hello` first
/// frame; non-response to our `Hello`), or an undecodable byte stream.
Protocol,
/// EOF before the handshake resolved. On the dial side this is also
/// what losing the tie-break looks like: the peer closes silently.
Closed,
/// [`HANDSHAKE_TIMEOUT`] (or the caller's deadline) passed first.
TimedOut,
/// The transport failed mid-handshake.
Transport(io::Error),
}
fn from_send(e: SendError) -> HandshakeError {
match e {
// Handshake frames are small and self-made; an encode failure is a
// protocol-level impossibility, not a transport fault.
SendError::Encode(_) => HandshakeError::Protocol,
SendError::Io(e) => HandshakeError::Transport(e),
}
}
fn from_recv(e: RecvError) -> HandshakeError {
match e {
RecvError::Corrupt(_) => HandshakeError::Protocol,
RecvError::TruncatedByPeer => HandshakeError::Closed,
RecvError::Io(e) => HandshakeError::Transport(e),
RecvError::TimedOut => HandshakeError::TimedOut,
}
}
/// Dial-side path step: send our `Hello`, interpret the one response. On
/// `Ok` the connection is established and `framed` is live (with any
/// read-ahead intact in its buffer); on `Err` the connection is closed.
pub fn dial_handshake(
framed: &mut FramedConn,
local: &Local,
deadline: Instant,
) -> Result<Peer, HandshakeError> {
let (initiator, hello) = Initiator::new(local);
if let Err(e) = framed.send(&hello) {
framed.close();
return Err(from_send(e));
}
let outcome = match framed.recv_deadline(deadline) {
Ok(Some(frame)) => initiator.on_frame(frame),
Ok(None) => {
framed.close();
return Err(HandshakeError::Closed);
}
Err(e) => {
framed.close();
return Err(from_recv(e));
}
};
match outcome {
InitiatorOutcome::Established(peer) => Ok(peer),
InitiatorOutcome::Rejected(reason) => {
framed.close();
Err(HandshakeError::Rejected(reason))
}
InitiatorOutcome::Failed(_) => {
framed.close();
Err(HandshakeError::Protocol)
}
}
}
/// Accept-side path step: read the first frame, judge it, answer or close.
///
/// `standing_of` supplies the [`PeerStanding`] of the *offered* name — knowledge
/// only the frame reveals, which is why it is a callback and not a value
/// (the integrated acceptor asks the manager; loopback tests fabricate).
/// It is not called when the first frame is not a `Hello`.
///
/// On `Ok` the ack has been sent and `framed` is live (read-ahead intact);
/// on `Err` any owed reject has been sent and the connection is closed.
pub fn accept_handshake(
framed: &mut FramedConn,
local: Local,
standing_of: impl FnOnce(&str) -> PeerStanding,
deadline: Instant,
) -> Result<Peer, HandshakeError> {
let frame = match framed.recv_deadline(deadline) {
Ok(Some(frame)) => frame,
Ok(None) => {
framed.close();
return Err(HandshakeError::Closed);
}
Err(e) => {
framed.close();
return Err(from_recv(e));
}
};
let standing = match &frame {
Frame::Hello { node_name, .. } => standing_of(node_name),
_ => PeerStanding::Free,
};
match Responder::new(local).on_frame(frame, standing) {
ResponderOutcome::Accepted { reply, peer } => {
if let Err(e) = framed.send(&reply) {
framed.close();
return Err(from_send(e));
}
Ok(peer)
}
ResponderOutcome::Rejected { reply, reason } => {
// Best effort: the reject is the cross-version compatibility
// anchor, but if the write fails the peer sees a bare close,
// which it must survive anyway.
let _ = framed.send(&reply);
framed.close();
Err(HandshakeError::Rejected(reason))
}
ResponderOutcome::TieBreakLoss => {
// D7: close silently — the peer computes the same verdict.
framed.close();
Err(HandshakeError::TieBreakLoss)
}
ResponderOutcome::Failed(_) => {
framed.close();
Err(HandshakeError::Protocol)
}
}
}
// ---------------------------------------------------------------------------
// Manager-integrated layer
// ---------------------------------------------------------------------------
/// Why an integrated [`dial`] did not produce a connection.
#[derive(Debug)]
pub enum DialError {
/// Another dial to this peer name is already in flight.
AlreadyDialing,
/// The manager is not running (or answered nonsense).
ManagerUnavailable,
/// The transport could not connect.
Connect(io::Error),
/// Connected, but the handshake did not establish.
Handshake(HandshakeError),
/// The peer at `addr` established, but answered as a different name
/// than the one we dialed — the tie-break bookkeeping (keyed by the
/// dialed name) would be unsound, so the connection is closed.
PeerNameMismatch { expected: String, got: String },
/// The handshake established, but the manager refused the registration:
/// a connection to this peer already exists. The loser has been closed.
Duplicate,
}
impl DialError {
/// A short static label per kind, for the `smarm-trace` `ClusterDial`
/// event; the payload (io error, names) is not carried.
pub fn label(&self) -> &'static str {
match self {
DialError::AlreadyDialing => "already_dialing",
DialError::ManagerUnavailable => "manager_unavailable",
DialError::Connect(_) => "connect",
DialError::Handshake(_) => "handshake",
DialError::PeerNameMismatch { .. } => "peer_name_mismatch",
DialError::Duplicate => "duplicate",
}
}
}
/// Dial `peer_name` at `addr` and run the handshake, keeping the manager's
/// dial-intent set honest around it: the intent is registered *before*
/// connecting (so a crossing inbound `Hello` sees it) and cleared the
/// moment the handshake resolves, before the connection actor is spawned.
/// Must run inside an actor. Retrying is the caller's business (c7's dial
/// loop); a lost tie-break surfaces as `Handshake(Closed)` — the peer's
/// accepted connection is already on its way.
pub fn dial(
transport: &dyn Transport,
addr: &str,
peer_name: &str,
local: &Local,
timing: Timing,
) -> Result<Pid, DialError> {
let me = scheduler::self_pid();
match gen_server::call(
MANAGER,
Call::DialBegin {
name: peer_name.to_string(),
pid: me,
},
) {
Ok(Reply::DialBegan(true)) => {}
Ok(Reply::DialBegan(false)) => return Err(DialError::AlreadyDialing),
_ => return Err(DialError::ManagerUnavailable),
}
let result = connect_and_shake(transport, addr, local, timing);
// Cleared immediately on outcome — a stale intent during the established
// window would corrupt later tie-breaks. Synchronous (a call): the
// intent is provably gone before anything else happens.
let _ = gen_server::call(
MANAGER,
Call::DialEnd {
name: peer_name.to_string(),
},
);
let (mut framed, peer) = result?;
if peer.node_name != peer_name {
framed.close();
return Err(DialError::PeerNameMismatch {
expected: peer_name.to_string(),
got: peer.node_name,
});
}
spawn_established(framed, peer, timing).map_err(|_| DialError::Duplicate)
}
fn connect_and_shake(
transport: &dyn Transport,
addr: &str,
local: &Local,
timing: Timing,
) -> Result<(FramedConn, Peer), DialError> {
let conn = transport.dial(addr).map_err(DialError::Connect)?;
let mut framed = FramedConn::new(conn);
let deadline = Instant::now() + timing.handshake_timeout;
let peer = dial_handshake(&mut framed, local, deadline).map_err(DialError::Handshake)?;
Ok((framed, peer))
}
/// A running acceptor. [`shutdown`](AcceptorHandle::shutdown) (or dropping
/// the last handle) stops the accept loop only: connections it established
/// belong to the [`manager`](crate::cluster::manager) and keep running, to
/// be torn down through the table (`Disconnect`, a peer close, or manager
/// shutdown).
pub struct AcceptorHandle {
cmd_tx: Sender<()>,
addr: String,
}
impl AcceptorHandle {
/// Ask the acceptor to stop. Idempotent; a no-op if it already has.
pub fn shutdown(&self) {
let _ = self.cmd_tx.send(());
}
/// The concrete bound address, dialable as-is.
pub fn local_addr(&self) -> &str {
&self.addr
}
}
/// Spawn the acceptor actor over a bound listener. Each inbound connection
/// is handshaken **inline in the loop** (a deliberate serialization: the
/// per-frame deadline bounds how long any one peer can hold the line, and
/// nothing concurrent exists to be starved before c7). The listener must be
/// fd-backed ([`Listener::readable_arm`]); the loopback listener is not,
/// and its acceptor exits immediately — loopback handshakes are driven
/// synchronously through the path fns instead, per D8.
pub fn spawn_acceptor(listener: Box<dyn Listener>, local: Local, timing: Timing) -> AcceptorHandle {
let addr = listener.local_addr();
let (cmd_tx, cmd_rx) = channel();
spawn(move || accept_loop(listener, local, timing, cmd_rx));
AcceptorHandle { cmd_tx, addr }
}
fn accept_loop(
mut listener: Box<dyn Listener>,
local: Local,
timing: Timing,
cmd_rx: Receiver<()>,
) {
loop {
let Some(arm) = listener.readable_arm() else {
return;
};
let arms: [&dyn Selectable; 2] = [&cmd_rx, &arm];
match crate::channel::try_select(&arms) {
Ok(0) => match cmd_rx.try_recv() {
Ok(Some(())) => return,
Ok(None) => continue, // spurious wake
Err(_) => return, // all handles dropped
},
Ok(_) => {
// The listener is readable: accept completes without parking.
let conn = match listener.accept() {
Ok(conn) => conn,
Err(_) => return, // listener itself is broken
};
handle_inbound(FramedConn::new(conn), &local, timing);
}
Err(_) => return, // fd arm failed to register: listener is gone
}
}
}
/// Run the accept-side handshake for one inbound connection, asking the
/// manager for the [`PeerStanding`], and hand the established connection to the
/// manager. Every failure was already resolved on the path (reject sent /
/// closed, or the registration refused and the actor stopped), so there is
/// nothing for the acceptor to carry forward.
fn handle_inbound(mut framed: FramedConn, local: &Local, timing: Timing) {
let deadline = Instant::now() + timing.handshake_timeout;
let standing_of = |name: &str| match gen_server::call(
MANAGER,
Call::Standing {
peer_name: name.to_string(),
},
) {
Ok(Reply::Standing(s)) => s,
// Manager unreachable: nobody could register this connection anyway,
// so claim the name taken and reject rather than accept an orphan.
_ => PeerStanding::Claimed,
};
if let Ok(peer) = accept_handshake(&mut framed, local.clone(), standing_of, deadline) {
let _ = spawn_established(framed, peer, timing);
}
}
+316
View File
@@ -0,0 +1,316 @@
//! RFC 010 c7b — the connector: the dial loop that turns discovered
//! candidates into a full mesh.
//!
//! A plain select-loop actor (the c6 shape). It spawns its [`Strategy`] as a
//! child actor and receives [`Discovery`] events from it; it tracks which
//! peers are up by **subscribing to membership like any other consumer** —
//! no privileged channel into the manager, the same snapshot-then-stream
//! surface c8 will use. One `select` folds the command inbox, the discovery
//! stream, the membership stream, and the earliest retry deadline into a
//! single wait.
//!
//! Per-candidate state: dial on arrival; on failure retry with capped
//! exponential backoff ([`INITIAL_BACKOFF`] doubling to [`MAX_BACKOFF`]);
//! on the peer's `node_up` stop dialing and reset the backoff; on its
//! `node_down` resume immediately (a fresh sequence — the reconnect case is
//! the one backoff exists to pace, but the *first* retry after a death
//! should be prompt). A candidate bearing our own name is parked permanently
//! — that seed is us; so is one whose address answers as a different name
//! (`PeerNameMismatch`: a misconfigured or stale seed — each retry would
//! only blip the peer's membership). Every other failure retries: in
//! particular a `NameTaken` reject can be our own ghost at the peer, not
//! yet reaped by its liveness timer, so it must not park. Each attempt's
//! outcome is one `smarm-trace` `ClusterDial` event. A [`Discovery::Withdrawn`]
//! drops its `(name, addr)` from the dial set — only that: a live
//! connection is membership's, and a re-announce re-adds it fresh.
//!
//! Dials run **inline in the loop** — the same deliberate serialization as
//! the acceptor (c6b): each attempt is bounded by the connect + handshake
//! deadlines, and nothing concurrent exists to be starved. A wall of slow
//! unreachable seeds would stretch the loop's latency; revisit if a real
//! deployment ever hits that shape.
use std::collections::HashSet;
use std::time::{Duration, Instant};
use crate::channel::{channel, select, select_timeout, Receiver, Selectable, Sender};
use crate::cluster::connect::{dial, DialError};
use crate::cluster::discovery::{Discovery, Strategy};
use crate::cluster::handshake::Local;
use crate::cluster::membership::{subscribe, NodeEvent};
use crate::cluster::transport::Transport;
use crate::cluster::Timing;
use crate::scheduler::spawn;
/// Default for [`Timing::initial_backoff`]: first retry delay after a failed
/// dial attempt.
pub const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
/// Default for [`Timing::max_backoff`]: an unreachable seed is retried this
/// often, forever.
pub const MAX_BACKOFF: Duration = Duration::from_secs(5);
enum Cmd {
Shutdown,
}
/// A running connector. `shutdown` (or dropping the last handle) stops the
/// dial loop and its strategy only — established connections belong to the
/// manager, exactly as with the acceptor.
pub struct ConnectorHandle {
cmd_tx: Sender<Cmd>,
}
impl ConnectorHandle {
/// Ask the connector to stop. Idempotent; a no-op if it already has.
pub fn shutdown(&self) {
let _ = self.cmd_tx.send(Cmd::Shutdown);
}
}
/// One discovered `(name, addr)` and our dial intent towards it.
struct Candidate {
name: String,
addr: String,
state: State,
}
/// The connector's *intent* for a candidate. Whether the peer is currently
/// up is a separate, name-keyed membership fact (`up` in [`run`]): a
/// candidate can arrive after its peer's `node_up` (the snapshot lands
/// before the strategy has said anything), so "up" cannot live on the
/// candidate alone — it is a filter over dialing, not a candidate state.
enum State {
/// Never dialed: this seed is the local node itself, or the address
/// answered as a *different* name than the one seeded
/// (`DialError::PeerNameMismatch` — a misconfigured or stale seed;
/// redialing would only blip the peer's membership forever). The way
/// back is the strategy's: `Withdrawn` then a fresh `Candidate`.
Parked,
/// Dial when due; on failure, back off.
Dialing {
/// Delay to apply after the *next* failure.
backoff: Duration,
next_attempt: Instant,
},
}
impl State {
fn fresh(timing: &Timing) -> Self {
State::Dialing {
backoff: timing.initial_backoff,
next_attempt: Instant::now(),
}
}
}
impl Candidate {
/// The retry deadline, if this candidate is dialing at all.
fn due(&self) -> Option<Instant> {
match self.state {
State::Parked => None,
State::Dialing { next_attempt, .. } => Some(next_attempt),
}
}
/// A dial attempt was made: schedule the retry, grow the backoff.
fn attempted(&mut self, timing: &Timing) {
if let State::Dialing {
backoff,
next_attempt,
} = &mut self.state
{
*next_attempt = Instant::now() + *backoff;
*backoff = (*backoff * 2).min(timing.max_backoff);
}
}
/// The peer came up: the next sequence (after a later `node_down`)
/// starts from the initial delay again.
fn peer_up(&mut self, timing: &Timing) {
if let State::Dialing { backoff, .. } = &mut self.state {
*backoff = timing.initial_backoff;
}
}
/// The peer went down: redial promptly, fresh sequence.
fn peer_down(&mut self, timing: &Timing) {
if matches!(self.state, State::Dialing { .. }) {
self.state = State::fresh(timing);
}
}
}
/// Spawn the connector actor. The strategy is spawned as its child; the
/// membership subscription is taken inside the actor. Must be called from
/// inside an actor (the same requirement as `dial`).
pub fn spawn_connector(
transport: Box<dyn Transport>,
local: Local,
strategy: Box<dyn Strategy>,
timing: Timing,
) -> ConnectorHandle {
let (cmd_tx, cmd_rx) = channel();
spawn(move || run(transport, local, strategy, timing, cmd_rx));
ConnectorHandle { cmd_tx }
}
fn run(
transport: Box<dyn Transport>,
local: Local,
strategy: Box<dyn Strategy>,
timing: Timing,
cmd_rx: Receiver<Cmd>,
) {
// Membership is the connector's source of truth for "who is up" — the
// snapshot seeds `up` before any candidate arrives.
let Some(events) = subscribe() else {
return; // no manager, no cluster to connect
};
let (disc_tx, disc_rx) = channel();
spawn(move || strategy.run(disc_tx));
let mut cands: Vec<Candidate> = Vec::new();
let mut up: HashSet<String> = HashSet::new();
let mut strategy_done = false;
loop {
// Drain every input, then act. Order does not matter: acting is
// idempotent against the resulting state.
match drain_cmd(&cmd_rx) {
Drained::Stop => return,
Drained::Open => {}
}
if !strategy_done {
strategy_done = drain_discoveries(&disc_rx, &local, &timing, &mut cands);
}
match drain_events(&events.rx, &timing, &mut up, &mut cands) {
Drained::Stop => return, // manager gone: the cluster is tearing down
Drained::Open => {}
}
// Dial everything due, inline (see the module docs on serialization).
let now = Instant::now();
for c in cands
.iter_mut()
.filter(|c| !up.contains(&c.name) && c.due().is_some_and(|d| d <= now))
{
// On success the manager's node_up is on its way and lands in
// `up` (backing off meanwhile keeps a racing re-attempt from
// spinning); every failure retries — see the module docs —
// except a peer-name mismatch, which parks the candidate.
let outcome = dial(&*transport, &c.addr, &c.name, &local, timing);
note_dial(&outcome);
match outcome {
Err(DialError::PeerNameMismatch { .. }) => c.state = State::Parked,
_ => c.attempted(&timing),
}
}
// Wait: until the earliest retry deadline among actionable
// candidates, or indefinitely if none is pending.
let deadline = cands
.iter()
.filter(|c| !up.contains(&c.name))
.filter_map(Candidate::due)
.min();
let mut arms: Vec<&dyn Selectable> = vec![&cmd_rx, &events.rx];
if !strategy_done {
arms.push(&disc_rx);
}
match deadline {
Some(d) => {
let wait = d.saturating_duration_since(Instant::now());
let _ = select_timeout(&arms, wait);
}
None => {
let _ = select(&arms);
}
}
}
}
enum Drained {
Open,
Stop,
}
fn drain_cmd(rx: &Receiver<Cmd>) -> Drained {
match rx.try_recv() {
Ok(Some(Cmd::Shutdown)) => Drained::Stop,
Ok(None) => Drained::Open,
Err(_) => Drained::Stop, // all handles dropped
}
}
/// Pull every pending discovery into the candidate set (deduplicated by
/// `(name, addr)`; a candidate bearing the local name is parked; a
/// `Withdrawn` removes its pair from the dial set and nothing else — see
/// [`Discovery::Withdrawn`]). Returns `true` once the strategy's channel
/// closes — it has said all it will.
fn drain_discoveries(
rx: &Receiver<Discovery>,
local: &Local,
timing: &Timing,
cands: &mut Vec<Candidate>,
) -> bool {
loop {
match rx.try_recv() {
Ok(Some(Discovery::Withdrawn { name, addr })) => {
cands.retain(|c| !(c.name == name && c.addr == addr));
}
Ok(Some(Discovery::Candidate { name, addr })) => {
if cands.iter().any(|c| c.name == name && c.addr == addr) {
continue;
}
let state = if name == local.node_name {
State::Parked
} else {
State::fresh(timing)
};
cands.push(Candidate { name, addr, state });
}
Ok(None) => return false,
Err(_) => return true, // strategy done; its candidates live on here
}
}
}
/// Fold pending membership events into `up`; each is also a transition on
/// that peer's candidates (see [`Candidate::peer_up`] / [`peer_down`]).
///
/// [`peer_down`]: Candidate::peer_down
fn drain_events(
rx: &Receiver<NodeEvent>,
timing: &Timing,
up: &mut HashSet<String>,
cands: &mut [Candidate],
) -> Drained {
loop {
match rx.try_recv() {
Ok(Some(NodeEvent::NodeUp(info))) => {
cands
.iter_mut()
.filter(|c| c.name == info.name)
.for_each(|c| c.peer_up(timing));
up.insert(info.name);
}
Ok(Some(NodeEvent::NodeDown(info))) => {
up.remove(&info.name);
cands
.iter_mut()
.filter(|c| c.name == info.name)
.for_each(|c| c.peer_down(timing));
}
Ok(None) => return Drained::Open,
Err(_) => return Drained::Stop,
}
}
}
/// Surface a dial outcome: one `smarm-trace` event, nothing else. The
/// connector's bookkeeping is decided by the caller.
fn note_dial(outcome: &Result<crate::pid::Pid, DialError>) {
#[cfg(feature = "smarm-trace")]
crate::te!(crate::trace::Event::ClusterDial(
outcome.as_ref().map_or_else(DialError::label, |_| "ok")
));
#[cfg(not(feature = "smarm-trace"))]
let _ = outcome;
}
+76
View File
@@ -0,0 +1,76 @@
//! RFC 010 c7b — peer discovery: the [`Strategy`] seam and the static-seeds
//! implementation.
//!
//! A strategy is **push-based and runs as its own actor**: the
//! [`connector`](crate::cluster::connector) spawns it with the sending end of
//! a channel, and the strategy emits [`Discovery`] events whenever it learns
//! something — once at startup for a static list, continuously for a future
//! mDNS/DNS strategy — for as long as it cares to run. Returning ends the
//! strategy actor; the candidates it pushed live on in the connector (the
//! connector owns all retry/backoff state, so a strategy never re-announces).
//!
//! A candidate is a **`(node_name, addr)` pair**, not a bare address: the
//! dial path and the D7 tie-break are keyed by peer *name* (the dial intent
//! must be registered before connecting so a crossing inbound `Hello` sees
//! it), so an anonymous dial would reintroduce exactly the
//! simultaneous-connect flap D7 exists to prevent. Discovery mechanisms know
//! names — that is what they discover.
use crate::channel::Sender;
/// A discovery event, as pushed by a [`Strategy`].
///
/// `Candidate` announces, `Withdrawn` retracts — the primitive pair. A
/// strategy that wants TTL semantics builds them on top (track its own
/// last-seen times, emit `Withdrawn` on expiry); the connector deliberately
/// has no clock of its own for candidates (D11: strategies never
/// re-announce, the connector owns retry). `#[non_exhaustive]` so more can
/// land without breaking strategies.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Discovery {
/// A peer worth dialing: its claimed node name and a dialable address.
Candidate { name: String, addr: String },
/// Stop dialing this `(name, addr)`. Dial-set only: a connection that
/// is already up is membership's business and is left alone; an
/// attempt in flight completes on its own; a later `Candidate` for the
/// same pair re-adds it with fresh backoff. Unknown pairs are ignored.
Withdrawn { name: String, addr: String },
}
/// A source of peers to dial. Implementations are spawned as actors by the
/// connector — see the module docs for the contract.
pub trait Strategy: Send + 'static {
/// Run the strategy: push [`Discovery`] events into `out` as they are
/// learned; return when done discovering (or when `out` reports closed —
/// the connector is gone). Runs inside an actor, so blocking
/// cooperatively is fine.
fn run(self: Box<Self>, out: Sender<Discovery>);
}
/// The static-seeds strategy: a fixed `(name, addr)` list, announced once.
#[derive(Debug, Clone, Default)]
pub struct StaticSeeds {
seeds: Vec<(String, String)>,
}
impl StaticSeeds {
pub fn new(seeds: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>) -> Self {
StaticSeeds {
seeds: seeds
.into_iter()
.map(|(n, a)| (n.into(), a.into()))
.collect(),
}
}
}
impl Strategy for StaticSeeds {
fn run(self: Box<Self>, out: Sender<Discovery>) {
for (name, addr) in self.seeds {
if out.send(Discovery::Candidate { name, addr }).is_err() {
return; // connector gone; nobody to discover for
}
}
}
}
+549
View File
@@ -0,0 +1,549 @@
//! RFC 010 c2 — the owned wire envelope.
//!
//! Every control-plane frame is `u32` little-endian length prefix (of tag +
//! body), `u8` tag, hand-encoded body. postcard appears in exactly one place:
//! the payload blob inside `Send`/`SendNamed`, via [`encode_payload`] /
//! [`decode_payload`] — the seam where a codec swap would land (RFC 010 §2).
//! Everything else is hand-rolled and wholly owned.
//!
//! Integers are little-endian. Strings are `u16` length + UTF-8 bytes.
//! Payload blobs are `u32` length + bytes. Enum-shaped fields
//! ([`RejectReason`], [`DownReason`]) are a single tag byte.
use crate::monitor::DownReason;
use crate::pg::Incarnation;
/// Wire protocol version, checked in the handshake (c5).
pub const PROTO_VERSION: u32 = 1;
/// Hard cap on the length prefix. The control plane never carries bulk data
/// (RFC 010 §5 — that is the jarred rkyv plane), so anything larger is
/// corruption or an attack, not a legitimate frame.
pub const MAX_FRAME_LEN: usize = 16 * 1024 * 1024;
/// Per-node metadata exchanged in the handshake (RFC 010 §1: not identity).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NodeMeta {
pub role: String,
pub region: String,
}
/// Why a `Hello` was rejected.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RejectReason {
/// Build hashes differ — not the same binary.
HashMismatch,
/// The offered node name is already claimed by a live peer.
NameTaken,
/// Wire protocol version mismatch.
ProtoVersion,
}
/// The control-plane frame inventory (RFC 010, *Implementation details*).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Frame {
Hello {
proto_version: u32,
build_hash: u64,
node_name: String,
incarnation: Incarnation,
meta: NodeMeta,
},
HelloAck {
node_name: String,
incarnation: Incarnation,
meta: NodeMeta,
},
HelloReject {
reason: RejectReason,
},
Heartbeat,
Send {
/// Target slot index (node is implicit in the connection, incarnation
/// is bound at handshake — RFC 010 §3).
index: u32,
generation: u32,
type_hash: u64,
payload: Vec<u8>,
},
SendNamed {
name: String,
type_hash: u64,
payload: Vec<u8>,
},
Monitor {
monitor_id: u64,
index: u32,
generation: u32,
},
Demonitor {
monitor_id: u64,
},
Down {
monitor_id: u64,
reason: RemoteDownReason,
},
}
/// Why a remotely-monitored actor is reported down: either the target's own
/// terminal [`DownReason`] as its node recorded it, or the *link* to that
/// node was lost (or absent) — which says nothing about the actor itself.
///
/// This is the cluster-side widening of `DownReason` (p5): `Disconnected`
/// is a fact about a connection, never about a local actor, so it lives
/// here rather than in the core enum — a local `Down` can never carry it,
/// and matches on `DownReason` stay exhaustive over actor outcomes only.
/// On the wire `Local(r)` uses `r`'s tag and `Disconnected` is tag 5,
/// bound since c11; no peer emits it today (a lost link is synthesized
/// locally), but the codec honours it both ways.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RemoteDownReason {
/// The target itself terminated; the peer reported this reason.
Local(DownReason),
/// The link to the target's node was lost or was never up.
Disconnected,
}
impl RemoteDownReason {
/// The actor's own reason, if this was not a link loss.
pub fn local(self) -> Option<DownReason> {
match self {
RemoteDownReason::Local(r) => Some(r),
RemoteDownReason::Disconnected => None,
}
}
}
impl From<DownReason> for RemoteDownReason {
fn from(r: DownReason) -> Self {
RemoteDownReason::Local(r)
}
}
// Frame tags. 0 is deliberately unassigned so an all-zero buffer never parses.
const TAG_HELLO: u8 = 1;
const TAG_HELLO_ACK: u8 = 2;
const TAG_HELLO_REJECT: u8 = 3;
const TAG_HEARTBEAT: u8 = 4;
const TAG_SEND: u8 = 5;
const TAG_SEND_NAMED: u8 = 6;
const TAG_MONITOR: u8 = 7;
const TAG_DEMONITOR: u8 = 8;
const TAG_DOWN: u8 = 9;
// RejectReason tags.
const REJ_HASH_MISMATCH: u8 = 1;
const REJ_NAME_TAKEN: u8 = 2;
const REJ_PROTO_VERSION: u8 = 3;
// DownReason tags. Do not reuse tags.
const DR_EXIT: u8 = 1;
const DR_PANIC: u8 = 2;
const DR_STOPPED: u8 = 3;
const DR_NOPROC: u8 = 4;
const DR_DISCONNECTED: u8 = 5;
// `Shutdown` never rides in a `Down` by contract (a target that honours the
// request exits normally) — the tag exists so the codec stays total.
const DR_SHUTDOWN: u8 = 6;
/// Frame could not be encoded. The output buffer is left exactly as it was.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncodeError {
/// tag + body exceed [`MAX_FRAME_LEN`].
FrameTooLarge { len: usize },
/// A string field exceeds `u16::MAX` bytes.
StringTooLong { len: usize },
}
impl core::fmt::Display for EncodeError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::FrameTooLarge { len } => {
write!(f, "frame body of {len} bytes exceeds MAX_FRAME_LEN")
}
Self::StringTooLong { len } => {
write!(f, "string field of {len} bytes exceeds u16::MAX")
}
}
}
}
impl std::error::Error for EncodeError {}
/// Frame could not be decoded. Everything here is *corruption* — "not enough
/// bytes yet" is the `Ok(None)` streaming case, never an error.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DecodeError {
/// The length prefix exceeds [`MAX_FRAME_LEN`].
FrameTooLarge { declared: usize },
/// The length prefix is zero — there is no tag byte.
EmptyFrame,
/// Unknown frame tag.
UnknownTag(u8),
/// Unknown tag for an enum-shaped field.
UnknownEnumTag { what: &'static str, tag: u8 },
/// A field ran past the declared frame end (the length prefix lied long,
/// or a length-carrying field inside the body lied).
Truncated,
/// Bytes were left over after the body (the length prefix lied short).
Trailing { extra: usize },
/// A string field was not valid UTF-8.
Utf8,
}
impl core::fmt::Display for DecodeError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::FrameTooLarge { declared } => {
write!(f, "declared frame length {declared} exceeds MAX_FRAME_LEN")
}
Self::EmptyFrame => write!(f, "zero-length frame (no tag byte)"),
Self::UnknownTag(t) => write!(f, "unknown frame tag {t}"),
Self::UnknownEnumTag { what, tag } => write!(f, "unknown {what} tag {tag}"),
Self::Truncated => write!(f, "frame body truncated mid-field"),
Self::Trailing { extra } => write!(f, "{extra} trailing bytes after frame body"),
Self::Utf8 => write!(f, "string field is not valid UTF-8"),
}
}
}
impl std::error::Error for DecodeError {}
impl Frame {
/// Append this frame, length-prefixed, to `out`.
///
/// On error `out` is left untouched.
pub fn encode(&self, out: &mut Vec<u8>) -> Result<(), EncodeError> {
let start = out.len();
out.extend_from_slice(&[0u8; 4]); // length placeholder, patched below
let result = self.encode_body(out);
match result {
Ok(()) => {
let frame_len = out.len() - start - 4;
if frame_len > MAX_FRAME_LEN {
out.truncate(start);
return Err(EncodeError::FrameTooLarge { len: frame_len });
}
// Cast is lossless: MAX_FRAME_LEN < u32::MAX, checked above.
let len32 = frame_len as u32;
out[start..start + 4].copy_from_slice(&len32.to_le_bytes());
Ok(())
}
Err(e) => {
out.truncate(start);
Err(e)
}
}
}
fn encode_body(&self, out: &mut Vec<u8>) -> Result<(), EncodeError> {
match self {
Frame::Hello {
proto_version,
build_hash,
node_name,
incarnation,
meta,
} => {
out.push(TAG_HELLO);
put_u32(out, *proto_version);
put_u64(out, *build_hash);
put_str(out, node_name)?;
put_u32(out, incarnation.get());
put_meta(out, meta)?;
}
Frame::HelloAck {
node_name,
incarnation,
meta,
} => {
out.push(TAG_HELLO_ACK);
put_str(out, node_name)?;
put_u32(out, incarnation.get());
put_meta(out, meta)?;
}
Frame::HelloReject { reason } => {
out.push(TAG_HELLO_REJECT);
out.push(match reason {
RejectReason::HashMismatch => REJ_HASH_MISMATCH,
RejectReason::NameTaken => REJ_NAME_TAKEN,
RejectReason::ProtoVersion => REJ_PROTO_VERSION,
});
}
Frame::Heartbeat => out.push(TAG_HEARTBEAT),
Frame::Send {
index,
generation,
type_hash,
payload,
} => {
out.push(TAG_SEND);
put_u32(out, *index);
put_u32(out, *generation);
put_u64(out, *type_hash);
put_blob(out, payload)?;
}
Frame::SendNamed {
name,
type_hash,
payload,
} => {
out.push(TAG_SEND_NAMED);
put_str(out, name)?;
put_u64(out, *type_hash);
put_blob(out, payload)?;
}
Frame::Monitor {
monitor_id,
index,
generation,
} => {
out.push(TAG_MONITOR);
put_u64(out, *monitor_id);
put_u32(out, *index);
put_u32(out, *generation);
}
Frame::Demonitor { monitor_id } => {
out.push(TAG_DEMONITOR);
put_u64(out, *monitor_id);
}
Frame::Down { monitor_id, reason } => {
out.push(TAG_DOWN);
put_u64(out, *monitor_id);
out.push(match reason {
RemoteDownReason::Local(DownReason::Exit) => DR_EXIT,
RemoteDownReason::Local(DownReason::Panic) => DR_PANIC,
RemoteDownReason::Local(DownReason::Stopped) => DR_STOPPED,
RemoteDownReason::Local(DownReason::NoProc) => DR_NOPROC,
RemoteDownReason::Local(DownReason::Shutdown) => DR_SHUTDOWN,
RemoteDownReason::Disconnected => DR_DISCONNECTED,
});
}
}
Ok(())
}
/// Try to decode one frame from the start of `buf`.
///
/// `Ok(Some((frame, consumed)))` — a full frame; the caller advances by
/// `consumed`. `Ok(None)` — not enough bytes yet (streaming); read more
/// and retry. `Err(_)` — the bytes are corrupt; the connection is dead.
pub fn decode(buf: &[u8]) -> Result<Option<(Frame, usize)>, DecodeError> {
let Some(prefix) = buf.get(0..4) else {
return Ok(None);
};
let mut len4 = [0u8; 4];
len4.copy_from_slice(prefix);
let declared = u32::from_le_bytes(len4) as usize;
if declared > MAX_FRAME_LEN {
return Err(DecodeError::FrameTooLarge { declared });
}
if declared == 0 {
return Err(DecodeError::EmptyFrame);
}
let Some(body) = buf.get(4..4 + declared) else {
return Ok(None);
};
let mut r = Reader { buf: body, pos: 0 };
let frame = Self::decode_body(&mut r)?;
if r.pos != body.len() {
return Err(DecodeError::Trailing {
extra: body.len() - r.pos,
});
}
Ok(Some((frame, 4 + declared)))
}
fn decode_body(r: &mut Reader<'_>) -> Result<Frame, DecodeError> {
let tag = r.u8()?;
let frame = match tag {
TAG_HELLO => Frame::Hello {
proto_version: r.u32()?,
build_hash: r.u64()?,
node_name: r.string()?,
incarnation: Incarnation::new(r.u32()?),
meta: r.meta()?,
},
TAG_HELLO_ACK => Frame::HelloAck {
node_name: r.string()?,
incarnation: Incarnation::new(r.u32()?),
meta: r.meta()?,
},
TAG_HELLO_REJECT => Frame::HelloReject {
reason: match r.u8()? {
REJ_HASH_MISMATCH => RejectReason::HashMismatch,
REJ_NAME_TAKEN => RejectReason::NameTaken,
REJ_PROTO_VERSION => RejectReason::ProtoVersion,
t => {
return Err(DecodeError::UnknownEnumTag {
what: "RejectReason",
tag: t,
})
}
},
},
TAG_HEARTBEAT => Frame::Heartbeat,
TAG_SEND => Frame::Send {
index: r.u32()?,
generation: r.u32()?,
type_hash: r.u64()?,
payload: r.blob()?,
},
TAG_SEND_NAMED => Frame::SendNamed {
name: r.string()?,
type_hash: r.u64()?,
payload: r.blob()?,
},
TAG_MONITOR => Frame::Monitor {
monitor_id: r.u64()?,
index: r.u32()?,
generation: r.u32()?,
},
TAG_DEMONITOR => Frame::Demonitor {
monitor_id: r.u64()?,
},
TAG_DOWN => Frame::Down {
monitor_id: r.u64()?,
reason: match r.u8()? {
DR_EXIT => RemoteDownReason::Local(DownReason::Exit),
DR_PANIC => RemoteDownReason::Local(DownReason::Panic),
DR_STOPPED => RemoteDownReason::Local(DownReason::Stopped),
DR_NOPROC => RemoteDownReason::Local(DownReason::NoProc),
DR_SHUTDOWN => RemoteDownReason::Local(DownReason::Shutdown),
DR_DISCONNECTED => RemoteDownReason::Disconnected,
t => {
return Err(DecodeError::UnknownEnumTag {
what: "RemoteDownReason",
tag: t,
})
}
},
},
t => return Err(DecodeError::UnknownTag(t)),
};
Ok(frame)
}
}
// ---------------------------------------------------------------------------
// Body writers
// ---------------------------------------------------------------------------
fn put_u32(out: &mut Vec<u8>, v: u32) {
out.extend_from_slice(&v.to_le_bytes());
}
fn put_u64(out: &mut Vec<u8>, v: u64) {
out.extend_from_slice(&v.to_le_bytes());
}
fn put_str(out: &mut Vec<u8>, s: &str) -> Result<(), EncodeError> {
let Ok(len) = u16::try_from(s.len()) else {
return Err(EncodeError::StringTooLong { len: s.len() });
};
out.extend_from_slice(&len.to_le_bytes());
out.extend_from_slice(s.as_bytes());
Ok(())
}
fn put_blob(out: &mut Vec<u8>, b: &[u8]) -> Result<(), EncodeError> {
let Ok(len) = u32::try_from(b.len()) else {
return Err(EncodeError::FrameTooLarge { len: b.len() });
};
out.extend_from_slice(&len.to_le_bytes());
out.extend_from_slice(b);
Ok(())
}
fn put_meta(out: &mut Vec<u8>, m: &NodeMeta) -> Result<(), EncodeError> {
put_str(out, &m.role)?;
put_str(out, &m.region)
}
// ---------------------------------------------------------------------------
// Body reader
// ---------------------------------------------------------------------------
struct Reader<'a> {
buf: &'a [u8],
pos: usize,
}
impl Reader<'_> {
fn take(&mut self, n: usize) -> Result<&[u8], DecodeError> {
let end = self.pos.checked_add(n).ok_or(DecodeError::Truncated)?;
let s = self.buf.get(self.pos..end).ok_or(DecodeError::Truncated)?;
self.pos = end;
Ok(s)
}
fn u8(&mut self) -> Result<u8, DecodeError> {
Ok(self.take(1)?[0])
}
fn u16(&mut self) -> Result<u16, DecodeError> {
let mut b = [0u8; 2];
b.copy_from_slice(self.take(2)?);
Ok(u16::from_le_bytes(b))
}
fn u32(&mut self) -> Result<u32, DecodeError> {
let mut b = [0u8; 4];
b.copy_from_slice(self.take(4)?);
Ok(u32::from_le_bytes(b))
}
fn u64(&mut self) -> Result<u64, DecodeError> {
let mut b = [0u8; 8];
b.copy_from_slice(self.take(8)?);
Ok(u64::from_le_bytes(b))
}
fn string(&mut self) -> Result<String, DecodeError> {
let len = self.u16()? as usize;
let bytes = self.take(len)?;
match core::str::from_utf8(bytes) {
Ok(s) => Ok(s.to_owned()),
Err(_) => Err(DecodeError::Utf8),
}
}
fn blob(&mut self) -> Result<Vec<u8>, DecodeError> {
let len = self.u32()? as usize;
Ok(self.take(len)?.to_vec())
}
fn meta(&mut self) -> Result<NodeMeta, DecodeError> {
Ok(NodeMeta {
role: self.string()?,
region: self.string()?,
})
}
}
// ---------------------------------------------------------------------------
// The postcard seam (RFC 010 §2) — the ONLY place payload bytes are produced
// or consumed. A codec swap lands here and nowhere else.
// ---------------------------------------------------------------------------
/// Payload (de)serialization failed at the codec seam.
#[derive(Debug)]
pub struct PayloadError(String);
impl core::fmt::Display for PayloadError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(f, "payload codec: {}", self.0)
}
}
impl std::error::Error for PayloadError {}
/// Serialize a payload value to the wire blob.
pub fn encode_payload<T: serde::Serialize + ?Sized>(value: &T) -> Result<Vec<u8>, PayloadError> {
postcard::to_allocvec(value).map_err(|e| PayloadError(e.to_string()))
}
/// Deserialize a payload value from the wire blob.
pub fn decode_payload<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, PayloadError> {
postcard::from_bytes(bytes).map_err(|e| PayloadError(e.to_string()))
}
+238
View File
@@ -0,0 +1,238 @@
//! RFC 010 c8 — explicit exposure: the node's remote surface, and the
//! fixed-seed type hash.
//!
//! Nothing local is remotely reachable by default (RFC §4 — "a gun needs a
//! safety"). [`expose`] marks a registered name remotely addressable and
//! registers `M`'s decoder under [`type_hash::<M>()`](type_hash);
//! [`expose_type`] registers only the decoder (the reply-to path: a
//! `RemotePid<A>` received in a message is sendable only if `A::Msg`'s
//! decoder was explicitly registered). The exposed set is the node's
//! visible, auditable remote surface ([`exposed_names`]).
//!
//! ## Where the state lives
//!
//! On `RuntimeInner`, the [`pg`](crate::pg) pattern: a leaf-locked table,
//! cfg-gated behind the `cluster` feature (zero-cost-when-off, per c1).
//! Chosen over manager-held state because c9's inbound decode consults it
//! per frame — a hot path that must not serialize every remote delivery
//! through one gen_server. The state resets with the runtime, like every
//! registry.
//!
//! ## The watchable fold (D3), against the code as it stands
//!
//! RFC §4: the exposed set is not a new registry — it folds into the
//! existing `watchable` machinery, one set, two set-sites (a pid crossing
//! the membrane, and expose). Reading the code: `register` **already
//! stamps every named holder watchable** ("no successfully-registered actor
//! can die unflagged", registry.rs), so an exposed *name*'s holder needs no
//! extra mark here — the guarantee holds by registration, and re-registration
//! after a holder's death re-stamps the new holder for free (a per-tenancy
//! mark taken at expose time could not do that). The cluster's own
//! `mark_watchable` set-site is therefore the **pid crossing the wire** —
//! serialization of a pid into a frame, c10 — the exact analog of the
//! membrane crossing. What lives here is only the name/type-level state
//! neither the registry nor the slot bits can carry: which names are
//! exposed, and how to decode each type hash.
//!
//! ## The hash
//!
//! [`type_hash`] is FNV-1a 64 (fixed seed: the FNV offset basis) over
//! `TypeId`, so it is a constant of the binary: stable across runs of the
//! same build — exactly the scope the build-hash handshake reduces the mesh
//! to — and deliberately *not* stable across builds (scope guard: no
//! cross-version wire compatibility). A collision between two exposed types
//! degrades to a decode error or a refused channel, never a misroute — the
//! local `SendError::NoChannel` guarantee survives the network (RFC §3).
//!
//! ## The decoder contract
//!
//! A decoder is **decode-and-deliver-to-pid**: it captures `M` (the one
//! typed site), decodes the payload, and hands the value to the target's
//! published channel via the registry's own dynamic send. Wire-name →
//! local-pid resolution deliberately stays *outside* — that is c9's single
//! resolution seam, and it calls [`decode_deliver`].
use std::any::TypeId;
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use crate::cluster::envelope::{decode_payload, PayloadError};
use crate::pid::{Name, Pid};
use crate::registry::{send_dyn, SendError};
use crate::scheduler::with_runtime;
/// The fixed-seed `TypeId` → `u64` hash: FNV-1a 64 over the `TypeId`'s hash
/// bytes, seeded with the FNV offset basis. A constant of the binary — see
/// the module docs for scope.
pub fn type_hash<M: 'static>() -> u64 {
let mut h = Fnv1a64::new();
TypeId::of::<M>().hash(&mut h);
h.finish()
}
/// FNV-1a 64 as a `Hasher`, so `TypeId` (opaque, `Hash`-only) can feed it.
/// Same constants as the const fns in [`crate::cluster`] (BUILD_HASH).
struct Fnv1a64(u64);
impl Fnv1a64 {
fn new() -> Self {
Fnv1a64(0xcbf2_9ce4_8422_2325)
}
}
impl Hasher for Fnv1a64 {
fn write(&mut self, bytes: &[u8]) {
for &b in bytes {
self.0 ^= b as u64;
self.0 = self.0.wrapping_mul(0x0000_0100_0000_01b3);
}
}
fn finish(&self) -> u64 {
self.0
}
}
/// Why a [`decode_deliver`] did not deliver. Payload-free mirror of the
/// registry's `SendError` where relevant — the caller (c9's inbound path)
/// has only bytes to give back, not a typed message.
#[derive(Debug)]
pub enum DeliverError {
/// No decoder is registered under this hash — the type was never
/// exposed here.
UnknownType,
/// The bytes did not decode as the registered type.
Decode(PayloadError),
/// The target actor is dead (or was never alive).
Dead,
/// The target is live but has no channel for this message type, or that
/// channel is closed — the `NoChannel` guarantee: a decoded value is
/// refused, never misrouted.
WrongChannel,
}
/// A registered decoder: decode `bytes` as the captured type and deliver to
/// `pid`'s published channel. `Arc`, so [`decode_deliver`] can clone it out
/// from under the exposure lock and call it lock-free — the decoder's
/// `send_dyn` takes the registry lock, and the two are mutual Leaves that
/// must never nest.
type Decoder = std::sync::Arc<dyn Fn(Pid, &[u8]) -> Result<(), DeliverError> + Send + Sync>;
/// The exposure state, one per runtime (a `RuntimeInner` field, pg-style).
pub(crate) struct ExposureState {
/// The exposed names: registry key → the type hash it expects.
exposed: HashMap<&'static str, u64>,
/// The decoders: type hash → decode-and-deliver.
decoders: HashMap<u64, Decoder>,
}
impl ExposureState {
pub(crate) fn new() -> Self {
ExposureState {
exposed: HashMap::new(),
decoders: HashMap::new(),
}
}
}
/// Mark `name` remotely addressable and register `M`'s decoder under its
/// type hash (so both name-sends and pid-sends of `M` work — RFC §4).
/// Returns the hash.
///
/// Exposure is a **name-level fact**, independent of who currently holds the
/// name (names late-bind: the registry re-resolves on every send, and c9's
/// seam resolves per delivery). Exposing an unregistered name is therefore
/// valid — deliveries fail with "unresolved" until someone registers it.
/// Idempotent. Must run inside [`run`](crate::run).
pub fn expose<M>(name: Name<M>) -> u64
where
M: serde::de::DeserializeOwned + Send + 'static,
{
let h = ensure_decoder::<M>();
with_runtime(|inner| {
inner.exposure.lock().exposed.insert(name.as_str(), h);
});
h
}
/// Register only `M`'s decoder (no name): the reply-to path. Returns the
/// hash. Idempotent. Must run inside [`run`](crate::run).
pub fn expose_type<M>() -> u64
where
M: serde::de::DeserializeOwned + Send + 'static,
{
ensure_decoder::<M>()
}
fn ensure_decoder<M>() -> u64
where
M: serde::de::DeserializeOwned + Send + 'static,
{
let h = type_hash::<M>();
with_runtime(|inner| {
inner
.exposure
.lock()
.decoders
.entry(h)
.or_insert_with(decoder::<M>);
});
h
}
/// The one typed site: decode as `M`, deliver via the registry's dynamic
/// send. See the module docs for the error mapping.
fn decoder<M>() -> Decoder
where
M: serde::de::DeserializeOwned + Send + 'static,
{
std::sync::Arc::new(|pid, bytes| {
let m: M = decode_payload(bytes).map_err(DeliverError::Decode)?;
send_dyn(pid, m).map_err(|e| match e {
SendError::Dead(_) | SendError::Unresolved(_) | SendError::NoMember(_) => {
DeliverError::Dead
}
SendError::NoChannel(_) | SendError::Closed(_) => DeliverError::WrongChannel,
})
})
}
/// The type hash `name` was exposed with, or `None` if it is not exposed.
/// Must run inside [`run`](crate::run).
pub fn exposed_hash(name: &str) -> Option<u64> {
with_runtime(|inner| inner.exposure.lock().exposed.get(name).copied())
}
/// Whether a decoder is registered under `hash`. Must run inside
/// [`run`](crate::run).
pub fn decoder_registered(hash: u64) -> bool {
with_runtime(|inner| inner.exposure.lock().decoders.contains_key(&hash))
}
/// Decode `bytes` under `hash`'s registered decoder and deliver to `pid`.
/// This is the delivery half c9's single resolution seam calls after it has
/// resolved a wire name to a local pid. Must run inside [`run`](crate::run).
pub fn decode_deliver(hash: u64, to: Pid, bytes: &[u8]) -> Result<(), DeliverError> {
// Clone the Arc under the lock, call outside it: the decoder's
// `send_dyn` takes the registry lock — a mutual Leaf with the exposure
// lock (the runtime asserts if Leaves nest). This also keeps unrelated
// deliveries uncoupled from a slow decode.
let d = with_runtime(|inner| inner.exposure.lock().decoders.get(&hash).cloned());
match d {
Some(d) => d(to, bytes),
None => Err(DeliverError::UnknownType),
}
}
/// The auditable remote surface: every exposed name and its type hash,
/// unordered. Must run inside [`run`](crate::run).
pub fn exposed_names() -> Vec<(&'static str, u64)> {
with_runtime(|inner| {
inner
.exposure
.lock()
.exposed
.iter()
.map(|(&n, &h)| (n, h))
.collect()
})
}
+178
View File
@@ -0,0 +1,178 @@
//! RFC 010 c5 — the handshake as a pure state machine.
//!
//! Frames in, actions out — no IO, no clocks, no actors. The c6 connection
//! actor drives these machines and executes their actions; everything
//! time-shaped (handshake deadline, heartbeats) lives there.
use crate::cluster::envelope::{Frame, NodeMeta, RejectReason, PROTO_VERSION};
use crate::pg::Incarnation;
/// This node's identity and metadata, as offered in (or checked against) a
/// `Hello`.
#[derive(Debug, Clone)]
pub struct Local {
pub node_name: String,
pub incarnation: Incarnation,
pub build_hash: u64,
pub meta: NodeMeta,
}
/// The peer identity a successful handshake yields (what c7 feeds `node_up`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Peer {
pub node_name: String,
pub incarnation: Incarnation,
pub meta: NodeMeta,
}
/// Driver-supplied standing of the *offered* name at this node — knowledge
/// the pure machine cannot have (c6 owns the connection table and dial
/// set). One answer, in the responder's own precedence: an established
/// peer under that name outranks an in-flight dial to it.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum PeerStanding {
/// Neither connected to nor dialing that name.
#[default]
Free,
/// An established peer already holds that name.
Claimed,
/// We have our own dial in flight to that name.
Dialing,
}
/// Simultaneous-connect tie-break: does the connection dialed by
/// `dialer_name` survive against the reverse dial?
/// The rule (ratified 2026-08-14, a wire-protocol fact): the connection
/// dialed by the lexicographically **smaller** name survives. Both ends know
/// both names, so both compute the same verdict — which is why the losing
/// side may close silently instead of sending a reject.
pub fn dial_wins(dialer_name: &str, acceptor_name: &str) -> bool {
dialer_name < acceptor_name
}
/// Dial side: emits `Hello` at construction, interprets the single response.
#[must_use]
#[derive(Debug)]
pub struct Initiator(());
/// What the dial side's response frame meant.
#[must_use]
#[derive(Debug, PartialEq, Eq)]
pub enum InitiatorOutcome {
Established(Peer),
Rejected(RejectReason),
/// Protocol violation before the ack — close. Carries the offending frame.
Failed(Frame),
}
impl Initiator {
/// Start a dial-side handshake: the returned frame is the `Hello` to
/// send; the returned machine is the right to interpret the response.
pub fn new(local: &Local) -> (Self, Frame) {
let hello = Frame::Hello {
proto_version: PROTO_VERSION,
build_hash: local.build_hash,
node_name: local.node_name.clone(),
incarnation: local.incarnation,
meta: local.meta.clone(),
};
(Initiator(()), hello)
}
/// Interpret the response. The `HelloAck` carries no hash or version —
/// the responder already checked ours against its own, and equality is
/// symmetric, so a one-sided check is sound.
pub fn on_frame(self, frame: Frame) -> InitiatorOutcome {
match frame {
Frame::HelloAck {
node_name,
incarnation,
meta,
} => InitiatorOutcome::Established(Peer {
node_name,
incarnation,
meta,
}),
Frame::HelloReject { reason } => InitiatorOutcome::Rejected(reason),
other => InitiatorOutcome::Failed(other),
}
}
}
/// Accept side: awaits exactly one `Hello`, answers or closes.
#[must_use]
#[derive(Debug)]
pub struct Responder {
local: Local,
}
/// What to do with an inbound connection's first frame.
#[must_use]
#[derive(Debug, PartialEq, Eq)]
pub enum ResponderOutcome {
/// Send the ack; the connection is established.
Accepted { reply: Frame, peer: Peer },
/// Send the reject, then close.
Rejected { reply: Frame, reason: RejectReason },
/// Lost the simultaneous-connect tie-break: close silently, no frame.
TieBreakLoss,
/// Protocol violation before Hello — close, no reply. Carries the frame.
Failed(Frame),
}
impl Responder {
pub fn new(local: Local) -> Self {
Responder { local }
}
/// Judge the connection's first frame. Check order is proto → hash →
/// name → tie-break: validity before identity. `HelloReject` is the
/// cross-version compatibility anchor, so a version-mismatched peer
/// still gets one.
pub fn on_frame(self, frame: Frame, standing: PeerStanding) -> ResponderOutcome {
let Frame::Hello {
proto_version,
build_hash,
node_name,
incarnation,
meta,
} = frame
else {
return ResponderOutcome::Failed(frame);
};
let reject = |reason| ResponderOutcome::Rejected {
reply: Frame::HelloReject { reason },
reason,
};
if proto_version != PROTO_VERSION {
return reject(RejectReason::ProtoVersion);
}
if build_hash != self.local.build_hash {
return reject(RejectReason::HashMismatch);
}
if node_name == self.local.node_name || standing == PeerStanding::Claimed {
return reject(RejectReason::NameTaken);
}
// Simultaneous connect: the inbound frame is the peer's dial. If our
// own in-flight dial wins instead, drop this one silently — the peer
// computes the same verdict (see `dial_wins`).
if standing == PeerStanding::Dialing && !dial_wins(&node_name, &self.local.node_name) {
return ResponderOutcome::TieBreakLoss;
}
ResponderOutcome::Accepted {
reply: Frame::HelloAck {
node_name: self.local.node_name,
incarnation: self.local.incarnation,
meta: self.local.meta,
},
peer: Peer {
node_name,
incarnation,
meta,
},
}
}
}
+295
View File
@@ -0,0 +1,295 @@
//! RFC 010 c6 — the cluster connection manager.
//!
//! One manager per runtime: the single registry of live peer connections and
//! the source of truth for whether a peer name is already claimed. The
//! accept/connect path registers each established connection here, handing
//! over its [`ConnHandle`] — **the manager owns connection lifetime**. A
//! connection lives as long as its table entry, so it neither outlives nor
//! dies with whichever actor happened to establish it. Registered actors are
//! also *monitored*, so the table self-heals on any exit path — a connection
//! that panics, is cancelled, or closes cleanly is removed without
//! cooperation from the dying actor.
//!
//! The manager also holds the **membership state** (c7a): `node_up` fires on
//! a successful registration and `node_down` on removal — they are derived
//! facts of the exact events this table already owns, so holding the view
//! here means no cross-actor race between "connection exists" and "node is
//! up". The consumer surface (event types, [`subscribe`], [`view`],
//! semantics) is [`membership`](crate::cluster::membership); no consumer
//! ever touches the table itself.
//!
//! The connector dial loop is c7b, built on top of both.
use std::collections::HashMap;
use crate::channel::Sender;
use crate::cluster::conn::ConnHandle;
use crate::cluster::handshake::{Peer, PeerStanding};
use crate::cluster::membership::{NodeEvent, NodeInfo};
use crate::cluster::remote::{bind_outbound, unbind_outbound};
use crate::gen_server::{GenServer, GenServerCtx, GenServerName, Watcher};
use crate::monitor::{monitor, Down};
use crate::pg::NodeId;
use crate::pid::Pid;
/// Well-known name of the singleton manager within a runtime. Connection
/// actors reach it by name rather than by a passed-around ref, so a restarted
/// manager is always found at the same key.
pub const MANAGER: GenServerName<Manager> = GenServerName::new("smarm.cluster.manager");
/// One live connection's entry: the actor running it, the handle whose
/// lifetime *is* the connection's (see the module docs), and the peer's
/// membership identity (what `node_up` announced and `node_down` will name).
struct ConnEntry {
pid: Pid,
info: NodeInfo,
_handle: ConnHandle,
}
/// The connection registry: peer name → the connection actor that owns that
/// peer's control connection. Plus the membership state layered on it (c7a):
/// subscribers, and the `(name, incarnation)` → [`NodeId`] memo.
pub struct Manager {
conns: HashMap<String, ConnEntry>,
/// In-flight dial intents: peer name -> the actor performing the dial.
/// Registered *before* connecting so a crossing inbound `Hello` sees it
/// ([`PeerStanding::Dialing`]); cleared the moment
/// the dial resolves, and — because the dialer is monitored — on the
/// dialer's death, so a panicking dial can never wedge the tie-break.
dials: HashMap<String, Pid>,
/// Membership subscribers; a closed channel is pruned on the next emit.
subscribers: Vec<Sender<NodeEvent>>,
/// The [`NodeId`] memo: a reconnect at the same incarnation keeps its id,
/// a restart (new incarnation) allocates a fresh one. Grows one entry per
/// distinct `(name, incarnation)` ever seen — unbounded in principle,
/// bounded in practice by restarts actually happening.
ids: HashMap<(String, u32), NodeId>,
/// Next id to allocate. Starts at 1: id 0 is
/// [`DEFAULT_NODE_ID`](crate::pg::DEFAULT_NODE_ID), the local node.
next_id: u32,
watcher: Option<Watcher<Manager>>,
}
impl Manager {
pub fn new() -> Self {
Manager {
conns: HashMap::new(),
dials: HashMap::new(),
subscribers: Vec::new(),
ids: HashMap::new(),
next_id: 1,
watcher: None,
}
}
/// The memoized id for `(name, incarnation)` — see the field docs.
fn node_id(&mut self, name: &str, incarnation: u32) -> NodeId {
*self
.ids
.entry((name.to_string(), incarnation))
.or_insert_with(|| {
let id = NodeId::new(self.next_id);
self.next_id += 1;
id
})
}
/// Deliver `event` to every live subscriber, pruning the dead: a closed
/// channel means the subscriber dropped its [`MembershipEvents`]
/// (crate::cluster::membership::MembershipEvents).
fn emit(&mut self, event: &NodeEvent) {
self.subscribers.retain(|tx| tx.send(event.clone()).is_ok());
}
}
impl Default for Manager {
fn default() -> Self {
Manager::new()
}
}
/// Outcome of a [`Call::Register`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Registered {
/// The name was free; this connection is now the peer of record.
Ok,
/// Another live connection already holds this name — the caller lost the
/// race (or is a duplicate) and must not run.
Duplicate,
}
/// Requests to the manager.
pub enum Call {
/// The path claims its peer's name for a freshly-established connection,
/// handing the manager the actor's [`ConnHandle`] and the handshake's
/// [`Peer`] (the membership identity `node_up` announces). The manager
/// monitors `pid` and holds the handle for as long as the entry lives; a
/// [`Registered::Duplicate`] verdict drops the handle here, which stops
/// the refused actor.
Register {
peer: Peer,
pid: Pid,
handle: ConnHandle,
},
/// Tear down the connection to `name`: the manager drops its handle, the
/// actor stops, and the monitor removes the entry. A no-op if no such
/// connection is live.
Disconnect { name: String },
/// The current peer names, sorted. For observation and tests.
Peers,
/// A dialer declares an in-flight dial to `name` before connecting. The
/// pid is the dialing actor, monitored so the intent dies with it.
DialBegin { name: String, pid: Pid },
/// The dial to `name` resolved (either way): drop the intent. A call,
/// not a cast, so the intent is provably gone before the dialer moves on.
DialEnd { name: String },
/// The [`PeerStanding`] of an inbound `Hello` offering `peer_name` — the
/// accept path asks this between reading the frame and judging it.
Standing { peer_name: String },
/// Subscribe `tx` to membership events, snapshot-then-stream: one
/// [`NodeEvent::NodeUp`] per live peer is queued into `tx` before this
/// call answers, so the stream is exact from its first event (handlers
/// are serialized — nothing interleaves with the snapshot). Use
/// [`subscribe`](crate::cluster::membership::subscribe).
Subscribe { tx: Sender<NodeEvent> },
/// The current view: every live peer's [`NodeInfo`], unordered. Use
/// [`view`](crate::cluster::membership::view).
View,
}
/// Replies from the manager.
#[derive(Debug)]
pub enum Reply {
Registered(Registered),
Disconnected,
Peers(Vec<String>),
/// `false`: another dial to this name is already in flight — do not dial.
DialBegan(bool),
DialEnded,
Standing(PeerStanding),
Subscribed,
View(Vec<NodeInfo>),
}
impl GenServer for Manager {
type Call = Call;
type Reply = Reply;
type Cast = ();
type Info = ();
type Timer = ();
fn init(&mut self, ctx: &GenServerCtx<Self>) {
self.watcher = Some(ctx.watcher());
}
/// Manager shutdown drops every entry (and with it every ConnHandle);
/// the outbound table must not outlive the connections it names.
fn terminate(&mut self) {
for name in self.conns.keys() {
unbind_outbound(name);
}
}
fn handle_call(&mut self, request: Call) -> Reply {
match request {
Call::Register {
peer,
pid,
mut handle,
} => {
if self.conns.contains_key(&peer.node_name) {
// `handle` drops here: the refused actor stops itself.
return Reply::Registered(Registered::Duplicate);
}
if let Some(w) = &self.watcher {
w.watch(monitor(pid));
}
// The outbound table (c9) is maintained here, inside the same
// serialized handlers that own the connection's lifetime.
if let Some((frames, monitors)) = handle.take_outbound() {
bind_outbound(&peer.node_name, peer.incarnation, frames, monitors);
}
let info = NodeInfo {
node: self.node_id(&peer.node_name, peer.incarnation.get()),
name: peer.node_name.clone(),
incarnation: peer.incarnation,
meta: peer.meta,
};
self.conns.insert(
peer.node_name,
ConnEntry {
pid,
info: info.clone(),
_handle: handle,
},
);
self.emit(&NodeEvent::NodeUp(info));
Reply::Registered(Registered::Ok)
}
Call::Disconnect { name } => {
// Dropping the entry drops the handle, which stops the actor.
if let Some(entry) = self.conns.remove(&name) {
unbind_outbound(&name);
self.emit(&NodeEvent::NodeDown(entry.info));
}
Reply::Disconnected
}
Call::Peers => {
let mut names: Vec<String> = self.conns.keys().cloned().collect();
names.sort();
Reply::Peers(names)
}
Call::DialBegin { name, pid } => {
if self.dials.contains_key(&name) {
return Reply::DialBegan(false);
}
if let Some(w) = &self.watcher {
w.watch(monitor(pid));
}
self.dials.insert(name, pid);
Reply::DialBegan(true)
}
Call::DialEnd { name } => {
self.dials.remove(&name);
Reply::DialEnded
}
Call::Standing { peer_name } => {
Reply::Standing(if self.conns.contains_key(&peer_name) {
PeerStanding::Claimed
} else if self.dials.contains_key(&peer_name) {
PeerStanding::Dialing
} else {
PeerStanding::Free
})
}
Call::Subscribe { tx } => {
// The snapshot: queued before `tx` joins the list, and — the
// handlers being serialized — before any later event.
for entry in self.conns.values() {
let _ = tx.send(NodeEvent::NodeUp(entry.info.clone()));
}
self.subscribers.push(tx);
Reply::Subscribed
}
Call::View => Reply::View(self.conns.values().map(|e| e.info.clone()).collect()),
}
}
fn handle_cast(&mut self, _request: ()) {}
fn handle_down(&mut self, down: Down) {
let mut downs = Vec::new();
self.conns.retain(|name, entry| {
let dead = entry.pid == down.pid;
if dead {
downs.push((name.clone(), entry.info.clone()));
}
!dead
});
for (name, info) in downs {
unbind_outbound(&name);
self.emit(&NodeEvent::NodeDown(info));
}
self.dials.retain(|_, pid| *pid != down.pid);
}
}
+97
View File
@@ -0,0 +1,97 @@
//! RFC 010 c7a — membership: `node_up`/`node_down` events and the view.
//!
//! The membership *state* lives inside the [`manager`](crate::cluster::manager)
//! — `node_up` and `node_down` are derived facts of the exact events the
//! manager already owns (a successful registration; a reap or `Disconnect`),
//! so holding the view anywhere else would only add a cross-actor ordering
//! seam. This module is the consumer surface: the event and view types, and
//! the [`subscribe`]/[`view`] entry points. No consumer ever touches the
//! connection table (roadmap-binding, enforced by module privacy: the table
//! is a private field, and nothing here exposes names→pids).
//!
//! ## Subscription semantics (ratified 2026-08-15)
//!
//! [`subscribe`] is **snapshot-then-stream**: the returned receiver first
//! yields one [`NodeEvent::NodeUp`] per currently-live peer, then live events
//! as they happen. Because the manager is a `gen_server` (handlers are
//! serialized), the snapshot is exact — no event can interleave with it, and
//! per-subscriber ordering matches the manager's processing order. There is
//! no join-race for late subscribers and no separate "get, then diff" dance;
//! [`view`] exists for observation, not for synchronization.
//!
//! A dropped subscriber is pruned on the next emission (its channel reports
//! closed) — no monitor needed, the sender itself tells us.
//!
//! ## NodeId identity
//!
//! A [`NodeId`] is a compact **local alias for the wire identity**
//! `(node_name, incarnation)`, memoized by the manager: a reconnect blip at
//! the same incarnation keeps its id (down, then up, same id), while a
//! restart — a new incarnation — gets a fresh one, so a node's ghost and its
//! successor are always distinguishable. Ids are allocated from 1;
//! [`DEFAULT_NODE_ID`](crate::pg::DEFAULT_NODE_ID) (0) remains the local
//! node, per [`pg`](crate::pg)'s framing.
use crate::channel::{channel, Receiver};
use crate::cluster::envelope::NodeMeta;
use crate::cluster::manager::{Call, Reply, MANAGER};
use crate::gen_server;
use crate::pg::{Incarnation, NodeId};
/// One live remote node, as the view and [`NodeEvent::NodeUp`] describe it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NodeInfo {
/// The local alias for `(name, incarnation)` — see the module docs.
pub node: NodeId,
/// The peer's claimed node name (handshake-verified).
pub name: String,
/// The peer's incarnation epoch, as offered in its `Hello`.
pub incarnation: Incarnation,
/// The peer's `Hello` metadata.
pub meta: NodeMeta,
}
/// A membership change, as delivered to subscribers.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum NodeEvent {
/// A peer's control connection established and registered.
NodeUp(NodeInfo),
/// That peer's connection ended — reaped, commanded down, or the manager
/// itself shut down. Carries the same [`NodeInfo`] the corresponding
/// `NodeUp` delivered, so consumers need no id→name reverse map.
NodeDown(NodeInfo),
}
/// A live membership subscription: the receiving end of the event stream
/// (the [`Monitor`](crate::monitor::Monitor) shape — read from [`rx`], drop
/// to unsubscribe).
///
/// [rx]: MembershipEvents::rx
pub struct MembershipEvents {
/// The event stream: the snapshot's `NodeUp`s first, then live events.
/// Fold it into a `select` from a plain actor, or pipe it into a
/// `gen_server` via `with_info`.
pub rx: Receiver<NodeEvent>,
}
/// Subscribe to membership events (snapshot-then-stream — see the module
/// docs). `None`: the manager is not running. Must be called from inside an
/// actor.
pub fn subscribe() -> Option<MembershipEvents> {
let (tx, rx) = channel();
match gen_server::call(MANAGER, Call::Subscribe { tx }) {
Ok(Reply::Subscribed) => Some(MembershipEvents { rx }),
_ => None,
}
}
/// The current view: every live peer's [`NodeInfo`], unordered. For
/// observation and tests; consumers that need to *track* the view should
/// [`subscribe`] instead (the snapshot makes the stream self-sufficient).
/// `None`: the manager is not running. Must be called from inside an actor.
pub fn view() -> Option<Vec<NodeInfo>> {
match gen_server::call(MANAGER, Call::View) {
Ok(Reply::View(v)) => Some(v),
_ => None,
}
}
+546
View File
@@ -0,0 +1,546 @@
//! RFC 010 c15 — distributed process groups (Phase 5).
//!
//! The Erlang `pg` shape (D18): every node's group store is the union of
//! its own local members and each peer's *announced* local members. There
//! is one **pg actor** per node — the c14 reaper grown up — and it is the
//! only writer of remote entries and the only sender of announcements:
//!
//! - **Origin owns its members.** Joins are local (`pg::join`), the eager
//! reaper is the liveness authority, and the origin announces every
//! change: `Join`/`Leave` incrementally to every up node, and a full
//! `Sync` of its local groups to a peer the moment that peer comes up
//! (`NodeUp`). Nobody monitors a remote member; a peer's `NodeDown` sweeps
//! every member it announced.
//! - **Transport is a pure consumer** of Phase 3/4: the exposed name
//! [`PG_NAME`] (`"pg"`) carrying [`PgMsg`] over postcard, sent with
//! [`remote::send`]. No new frame, no manager change.
//! - **No anti-entropy.** Per-origin ordering rides the single TCP link:
//! the actor sends `Sync` to a peer *before* it can send that peer any
//! `Join`/`Leave` (both from the same loop, over the same connection), and
//! a reconnect is a fresh `NodeUp` ⇒ fresh `Sync` replacing that peer's
//! set wholesale.
//! - **Local API unchanged.** `members`/`pick`/`dispatch` stay local-only
//! (`get_local_members`); a remote entry in the store carries the peer's
//! `NodeId` and never surfaces there. Cluster-wide reads are the new,
//! additive [`members_all`] over [`GroupMember`] (c16 adds `pick_any` /
//! `dispatch_any`).
//!
//! ## Ordering inside the node
//!
//! `pg::join`/`pg::leave` mutate the store on the caller's thread and then
//! *announce* to the actor's control inbox. Because the store op precedes the
//! announcement and the actor re-reads the store before broadcasting a
//! `Joined`, an announcement that has been overtaken (the member left or died
//! before the actor got to it) is dropped rather than advertised: the wire
//! never sees a `Join` for a member the origin no longer holds. `Leave`
//! broadcasts unconditionally — a spurious `Leave` is a no-op at the peer.
//!
//! Inbound: `NodeUp` is emitted by the manager on the accept/connect path,
//! *before* the peer's connection actor exists, so it is queued on the
//! membership stream before any frame from that peer can reach this inbox.
//! The actor still drains membership before it interprets a `PgMsg` whose
//! sender it does not know, and drops the message if the sender is still not
//! up (a ghost — its next `NodeUp` brings a `Sync`).
use std::collections::HashMap;
use crate::channel::{channel, select, Receiver, Selectable};
use crate::cluster::expose::expose;
use crate::cluster::membership::{subscribe, MembershipEvents, NodeEvent, NodeInfo};
use crate::cluster::remote::{
self, local_identity, send_to_remote, RemoteName, RemotePid, ToRemoteError,
};
use crate::monitor::Down;
use crate::pg::{
live, member_for, reaper_inboxes, sweep_local_death, Incarnation, Member, Membership, PgEvent,
};
use crate::pid::{assert_type, Addressable, Erased, Pid};
use crate::registry::{register, send_to, SendError};
use crate::scheduler::with_runtime;
use crate::Name;
/// The exposed name every node's pg actor answers under.
pub const PG_NAME: Name<PgMsg> = Name::new("pg");
/// The pg wire protocol. Every variant is origin-authored: `from` / the
/// pid's node is the node whose local members are being described.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PgMsg {
/// The origin's complete local membership, sent to a peer on `NodeUp`.
/// Replaces whatever the receiver held for that origin.
Sync {
from: String,
groups: Vec<(String, Vec<RemotePid<Erased>>)>,
},
/// The origin added `pid` (its own) to `group`.
Join {
group: String,
pid: RemotePid<Erased>,
},
/// The origin removed `pid` from `group` — voluntary leave or death.
Leave {
group: String,
pid: RemotePid<Erased>,
},
}
// Hand-rolled serde (the crate carries no serde-derive), as a 3-tuple with a
// leading tag: (0, from, groups) | (1, group, pid) | (2, group, pid).
impl serde::Serialize for PgMsg {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(3)?;
match self {
PgMsg::Sync { from, groups } => {
t.serialize_element(&0u8)?;
t.serialize_element(from)?;
t.serialize_element(groups)?;
}
PgMsg::Join { group, pid } => {
t.serialize_element(&1u8)?;
t.serialize_element(group)?;
t.serialize_element(pid)?;
}
PgMsg::Leave { group, pid } => {
t.serialize_element(&2u8)?;
t.serialize_element(group)?;
t.serialize_element(pid)?;
}
}
t.end()
}
}
impl<'de> serde::Deserialize<'de> for PgMsg {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
struct V;
impl<'de> serde::de::Visitor<'de> for V {
type Value = PgMsg;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a pg message tuple")
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(
self,
mut seq: A,
) -> Result<PgMsg, A::Error> {
use serde::de::Error;
let tag: u8 = seq
.next_element()?
.ok_or_else(|| A::Error::custom("pg: missing tag"))?;
let text: String = seq
.next_element()?
.ok_or_else(|| A::Error::custom("pg: missing name"))?;
match tag {
0 => {
let groups = seq
.next_element()?
.ok_or_else(|| A::Error::custom("pg: missing groups"))?;
Ok(PgMsg::Sync { from: text, groups })
}
1 | 2 => {
let pid = seq
.next_element()?
.ok_or_else(|| A::Error::custom("pg: missing pid"))?;
Ok(if tag == 1 {
PgMsg::Join { group: text, pid }
} else {
PgMsg::Leave { group: text, pid }
})
}
t => Err(A::Error::custom(format!("pg: unknown tag {t}"))),
}
}
}
d.deserialize_tuple(3, V)
}
}
/// A member of a group as the cluster sees it: on this node (a plain
/// [`Pid`], sendable locally) or on a peer (a [`RemotePid`], sendable via
/// [`send_to_remote`](remote::send_to_remote)). `Pid` cannot hold a remote
/// (D14), hence the two-variant shape.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GroupMember {
Local(Pid),
Remote(RemotePid<Erased>),
}
/// Every member of `group` cluster-wide, in the store's order: local members
/// filtered by the same liveness backstop as [`members`](crate::pg::members),
/// remote members exactly as their origins last announced them. Must run
/// inside [`run`](crate::run).
pub fn members_all(group: &str) -> Vec<GroupMember> {
with_runtime(|inner| {
let me = inner.node_id;
let pg = inner.process_groups.lock();
pg.all_of(group)
.into_iter()
.filter_map(|m| {
if m.node == me {
live(inner, m.pid).then_some(GroupMember::Local(m.pid))
} else {
// A remote entry always has its node's name recorded
// (they land under the same lock); a missing one is a
// node already swept, so it hides rather than misnames.
pg.node_name(m.node).map(|name| {
GroupMember::Remote(RemotePid::from_parts(
name,
m.incarnation,
m.pid.index(),
m.pid.generation(),
))
})
}
})
.collect()
})
}
/// One member of `group` cluster-wide, or `None` if it has none: the first
/// entry in the store's order (this node's members in join order first when
/// they joined first — the same stateless first-live scan as
/// [`pick`](crate::pg::pick), extended over the peers' announced members).
/// Must run inside [`run`](crate::run).
pub fn pick_any(group: &str) -> Option<GroupMember> {
members_all(group).into_iter().next()
}
/// Why [`dispatch_any`] handed `msg` back.
#[derive(Debug)]
pub enum DispatchAnyError<M> {
/// The group has no member anywhere.
NoMember(M),
/// The pick was local and the local typed send failed.
Local(SendError<M>),
/// The pick was remote and the remote send failed at this node.
Remote(ToRemoteError<M>),
}
impl<M> DispatchAnyError<M> {
/// The undelivered message.
pub fn into_inner(self) -> M {
match self {
DispatchAnyError::NoMember(m) => m,
DispatchAnyError::Local(e) => e.into_inner(),
DispatchAnyError::Remote(e) => e.into_inner(),
}
}
}
/// [`pick_any`] and send in one step, returning the member reached: a local
/// pick goes through [`send_to`], a remote one through [`send_to_remote`]
/// (so `Ok` for a remote member means "handed to the connection", RFC 010
/// §3). Homogeneous pool assumed, as for [`dispatch`](crate::pg::dispatch);
/// a wrong `A` degrades to a clean error at the target, never a misroute.
/// Must run inside [`run`](crate::run).
pub fn dispatch_any<A>(group: &str, msg: A::Msg) -> Result<GroupMember, DispatchAnyError<A::Msg>>
where
A: Addressable,
A::Msg: serde::Serialize,
{
match pick_any(group) {
None => Err(DispatchAnyError::NoMember(msg)),
Some(GroupMember::Local(pid)) => send_to(assert_type::<A>(pid), msg)
.map(|()| GroupMember::Local(pid))
.map_err(DispatchAnyError::Local),
Some(GroupMember::Remote(rp)) => send_to_remote(rp.clone().assert_type::<A>(), msg)
.map(|()| GroupMember::Remote(rp))
.map_err(DispatchAnyError::Remote),
}
}
/// Attach the pg actor to the running cluster. Called once by
/// `cluster::start` after the manager is up and the local identity is set;
/// spawns the actor if this run has not joined anything yet.
pub(crate) fn attach_cluster() {
let _ = reaper_inboxes().ctl.send(PgEvent::Attach);
}
/// The attached half of the actor's state: who is up (by name) and the
/// membership stream.
struct Attached {
events: MembershipEvents,
peers: HashMap<String, NodeInfo>,
/// This node's wire identity: `Sync`'s `from`, and the stamp on every
/// pid we ship (attach requires it, so no `None` path exists here).
me: String,
incarnation: Incarnation,
}
/// The pg actor: the c14 reaper (`deaths`), the local API's announcements
/// (`ctl`), and — once attached — the membership stream and the exposed
/// `"pg"` inbox, all in one drain-then-select loop. `deaths`/`ctl` closing
/// is the run tearing down; the membership stream closing is the manager
/// gone (detach, keep reaping).
pub(crate) fn actor(deaths: Receiver<Down>, ctl: Receiver<PgEvent>) {
let (pg_tx, pg_rx) = channel::<PgMsg>();
let mut cl: Option<Attached> = None;
loop {
loop {
match deaths.try_recv() {
Ok(Some(down)) => on_death(cl.as_ref(), down.pid),
Ok(None) => break,
Err(_) => return,
}
}
loop {
match ctl.try_recv() {
Ok(Some(PgEvent::Attach)) => {
// Own the name BEFORE subscribing (which yields to the
// manager): a peer's first frame must find "pg" exposed
// and resolvable, or it is dropped. Idempotent for a
// re-attach: same actor, same channel (the registry
// refuses a *second* live one).
let _ = register(PG_NAME, pg_tx.clone());
expose(PG_NAME);
if let Some(a) = attach() {
cl = Some(a);
}
}
Ok(Some(PgEvent::Joined { group, pid })) => on_joined(cl.as_ref(), &group, pid),
Ok(Some(PgEvent::Left { group, pid })) => on_left(cl.as_ref(), &group, pid),
Ok(None) => break,
Err(_) => return,
}
}
if let Some(a) = cl.as_mut() {
if !drain_events(a) {
cl = None;
continue;
}
loop {
match pg_rx.try_recv() {
Ok(Some(msg)) => on_msg(a, msg),
Ok(None) => break,
Err(_) => return, // our own inbox: only on teardown
}
}
}
// Wait. Control first (attach/teardown must be prompt), then deaths,
// then the cluster arms.
let mut arms: Vec<&dyn Selectable> = vec![&ctl, &deaths];
if let Some(a) = cl.as_ref() {
arms.push(&a.events.rx);
arms.push(&pg_rx);
}
let _ = select(&arms);
}
}
fn attach() -> Option<Attached> {
let events = subscribe()?;
let (me, incarnation) = local_identity()?;
Some(Attached {
events,
peers: HashMap::new(),
me,
incarnation,
})
}
/// Fold pending membership events: `NodeUp` ⇒ record + `Sync` that peer;
/// `NodeDown` ⇒ sweep every member it announced. `false` when the stream
/// has closed.
fn drain_events(a: &mut Attached) -> bool {
loop {
match a.events.rx.try_recv() {
Ok(Some(NodeEvent::NodeUp(info))) => {
let name = info.name.clone();
a.peers.insert(name.clone(), info);
// Snapshot under the store lock, then stamp wire pids
// outside it (`from_local` marks watchable under the slot's
// cold lock — Leaf-on-Leaf nesting is asserted).
let local: Vec<(String, Vec<Pid>)> =
with_runtime(|inner| inner.process_groups.lock().groups_on(inner.node_id));
let groups = local
.into_iter()
.map(|(g, pids)| (g, pids.into_iter().map(|p| wire(a, p)).collect()))
.collect();
let msg = PgMsg::Sync {
from: a.me.clone(),
groups,
};
let _ = remote::send(RemoteName::new(name, PG_NAME), msg);
}
Ok(Some(NodeEvent::NodeDown(info))) => {
a.peers.remove(&info.name);
with_runtime(|inner| {
let mut pg = inner.process_groups.lock();
pg.remove_where(|m| m.node == info.node);
pg.forget_node_name(info.node);
});
}
Ok(None) => return true,
Err(_) => return false,
}
}
}
/// Send `msg` to every up peer. `NotConnected` is ignored: that peer's
/// `NodeDown` is on its way and its next `NodeUp` gets a `Sync`.
fn broadcast(a: &Attached, msg: PgMsg) {
for name in a.peers.keys() {
let _ = remote::send(RemoteName::new(name.clone(), PG_NAME), msg.clone());
}
}
fn on_death(a: Option<&Attached>, pid: Pid) {
let evicted = sweep_local_death(pid);
if let Some(a) = a {
for (group, ms) in evicted {
broadcast(
a,
PgMsg::Leave {
group,
pid: wire(a, ms.member.pid),
},
);
}
}
}
fn on_joined(a: Option<&Attached>, group: &str, pid: Pid) {
let Some(a) = a else { return };
// Re-check: a leave/death may have overtaken the announcement.
let still = with_runtime(|inner| {
let m = member_for(inner, pid);
inner.process_groups.lock().contains(group, &m)
});
if still {
broadcast(
a,
PgMsg::Join {
group: group.to_owned(),
pid: wire(a, pid),
},
);
}
}
fn on_left(a: Option<&Attached>, group: &str, pid: Pid) {
let Some(a) = a else { return };
broadcast(
a,
PgMsg::Leave {
group: group.to_owned(),
pid: wire(a, pid),
},
);
}
/// The wire form of a local member pid, stamped with the identity the
/// actor was attached with (marks watchable, like `from_local`).
fn wire(a: &Attached, pid: Pid) -> RemotePid<Erased> {
RemotePid::from_local_at(pid, a.me.clone(), a.incarnation)
}
/// The named origin's `NodeInfo`, if it is up. A second look at the
/// membership stream covers a `NodeUp` that landed after this loop
/// iteration's drain; anything still unknown is a ghost and is dropped.
fn origin(a: &mut Attached, name: &str) -> Option<NodeInfo> {
if let Some(i) = a.peers.get(name) {
return Some(i.clone());
}
drain_events(a);
a.peers.get(name).cloned()
}
/// `origin`, additionally requiring `pid` to be stamped with the origin's
/// current incarnation — a pid from a previous life of that node is a ghost.
fn origin_of(a: &mut Attached, pid: &RemotePid<Erased>) -> Option<NodeInfo> {
origin(a, pid.node()).filter(|i| i.incarnation == pid.incarnation())
}
fn remote_membership(origin: &NodeInfo, pid: &RemotePid<Erased>) -> Membership {
Membership {
member: Member {
node: origin.node,
incarnation: origin.incarnation,
pid: Pid::new(pid.index(), pid.generation()),
},
monitor: None,
}
}
fn on_msg(a: &mut Attached, msg: PgMsg) {
match msg {
PgMsg::Sync { from, groups } => {
let Some(info) = origin(a, &from) else { return };
with_runtime(|inner| {
let mut pg = inner.process_groups.lock();
pg.remove_where(|m| m.node == info.node);
pg.set_node_name(info.node, info.name.clone());
for (group, pids) in &groups {
// Origin-authored: only its own current-incarnation pids.
for p in pids
.iter()
.filter(|p| p.node() == from && p.incarnation() == info.incarnation)
{
pg.join(group, remote_membership(&info, p));
}
}
});
}
PgMsg::Join { group, pid } => {
let Some(info) = origin_of(a, &pid) else {
return;
};
with_runtime(|inner| {
let mut pg = inner.process_groups.lock();
pg.set_node_name(info.node, info.name.clone());
pg.join(&group, remote_membership(&info, &pid));
});
}
PgMsg::Leave { group, pid } => {
let Some(info) = origin_of(a, &pid) else {
return;
};
with_runtime(|inner| {
let ms = remote_membership(&info, &pid);
inner.process_groups.lock().leave(&group, ms.member);
});
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cluster::envelope::{decode_payload, encode_payload};
#[test]
fn pg_msg_roundtrips_every_variant() {
let p = RemotePid::<Erased>::from_parts("a", Incarnation::new(9), 3, 1);
for m in [
PgMsg::Sync {
from: "a".into(),
groups: vec![
("g".into(), vec![p.clone(), p.clone()]),
("h".into(), vec![]),
],
},
PgMsg::Sync {
from: "a".into(),
groups: vec![],
},
PgMsg::Join {
group: "g".into(),
pid: p.clone(),
},
PgMsg::Leave {
group: "g".into(),
pid: p.clone(),
},
] {
let bytes = encode_payload(&m).unwrap();
let back: PgMsg = decode_payload(&bytes).unwrap();
assert_eq!(back, m);
}
}
#[test]
fn pg_msg_rejects_unknown_tag() {
let bytes = encode_payload(&(7u8, "x", 0u32)).unwrap();
assert!(decode_payload::<PgMsg>(&bytes).is_err());
}
}
+835
View File
@@ -0,0 +1,835 @@
//! RFC 010 c9 — remote `Name` sends: the outbound path and the single
//! inbound name-resolution seam.
//!
//! ## Outbound (D13, ratified 2026-08-15)
//!
//! A module-private table `node name → Sender<Frame>` — one dedicated
//! outbound channel per live connection, populated and torn down by the
//! manager inside the same serialized handlers that own the connection's
//! lifetime (Register / Disconnect / reap), living on `RuntimeInner` beside
//! the exposure state. [`send`] is one leaf-lock lookup + one channel send:
//! no gen_server on the data plane, no published channel anyone holding a
//! pid could inject raw frames into. `Ok(())` means **handed to the
//! connection's inbox** — local knowledge only, exactly the BEAM contract
//! (RFC §3): a missing entry or a closed channel is
//! [`RemoteSendError::NotConnected`]; delivery confirmation is the monitor's
//! job (c11). The entry-present/actor-dying-mid-send window is *honest*
//! under that contract, not a bug.
//!
//! The outbound sender is deliberately **separate from the conn actor's
//! command channel**: if it were a clone of `cmd_tx`, the manager dropping
//! its `ConnHandle` would no longer close that channel and connection
//! lifetime would leak to whoever holds a sender — a D9 violation.
//!
//! Buffering is unbounded toward a slow peer (the BEAM `busy_dist_port`
//! shape); backpressure is out of c9's scope and noted here rather than
//! silently absent.
//!
//! ## Inbound — the ONE resolution seam (RFC v2)
//!
//! Every wire-name → local-pid resolution goes through [`deliver_named`],
//! and nothing else: the conn actor hands it the three fields of a
//! `SendNamed` and gets back a verdict. It checks the exposed set first (an
//! unexposed name is unreachable — the gun's safety), then the type hash
//! against what the name was exposed with, then resolves the name through
//! the registry and delivers via c8's [`decode_deliver`]. When an owned-name
//! table lands beside the `&'static str` registry, it slots in here without
//! touching call sites. Module privacy enforces the funnel: the exposed and
//! outbound tables are `pub(crate)`, and no other module resolves names for
//! the wire.
//!
//! Refusals are silent to the sender by design (§3: send failure reflects
//! local knowledge only); they are observable locally as the returned
//! [`InboundVerdict`], which the conn actor may log or count.
//!
//! ## Pids (c10, D14)
//!
//! [`RemotePid<A>`] = `(node_name, incarnation, index, generation)` +
//! phantom — identity-bound, dead when that incarnation dies, never
//! redirects. The node travels as its **name** (a global identifier, so a pid
//! forwarded through a third node needs no re-mapping); NodeId is a local
//! alias and never crosses. A local `Pid<A>` serializes *as* a `RemotePid`
//! stamped from the ambient [local identity](set_local_identity); a
//! `RemotePid` deserializes into `Pid<A>` only when it names this node (the
//! collapse), else it is a decode error — fields that may hold a pid from
//! anywhere are typed `RemotePid<A>`.
//!
//! [`send_to_remote`] is the pid-targeted send. A self-node pid short-
//! circuits to the local typed send with the message object itself — no
//! encode, no frame (zero-copy-equivalent). Otherwise the outbound table
//! (widened to carry each node's **current incarnation**) does the RFC v2 §3
//! check at the send site: a pid of a dead incarnation is
//! [`ToRemoteError::DeadIncarnation`] and no frame is emitted. Inbound
//! `Send` frames are delivered by index/generation through c8's
//! [`decode_deliver`]: the target actor's published channel for the exposed
//! type is the only route (the reply-to path requires
//! [`expose_type`](crate::cluster::expose::expose_type) at the receiver).
use std::cell::Cell;
use std::collections::HashMap;
use std::marker::PhantomData;
use crate::channel::{channel, Receiver, RecvError, Selectable, Sender};
use crate::cluster::envelope::{encode_payload, Frame, PayloadError, RemoteDownReason};
use crate::cluster::expose::{decode_deliver, exposed_hash, type_hash, DeliverError};
use crate::monitor::{demonitor, monitor, Monitor, MonitorId};
use crate::pg::Incarnation;
use crate::pid::{Addressable, Erased, Name, Pid};
use crate::registry::{send_to, whereis, SendError};
use crate::scheduler::with_runtime;
/// A name on a specific remote node: `(node_name, Name<M>)`. Sendable via
/// [`send`]; typed, so the payload is `M` and the wire hash is
/// [`type_hash::<M>()`](type_hash).
pub struct RemoteName<M> {
node: String,
name: Name<M>,
_marker: PhantomData<fn() -> M>,
}
impl<M> RemoteName<M> {
pub fn new(node: impl Into<String>, name: Name<M>) -> Self {
RemoteName {
node: node.into(),
name,
_marker: PhantomData,
}
}
pub fn node(&self) -> &str {
&self.node
}
pub fn name(&self) -> Name<M> {
self.name
}
}
impl<M> Clone for RemoteName<M> {
fn clone(&self) -> Self {
RemoteName {
node: self.node.clone(),
name: self.name,
_marker: PhantomData,
}
}
}
impl<M> std::fmt::Debug for RemoteName<M> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}@{}", self.name.as_str(), self.node)
}
}
/// Why a remote send did not leave this node. Local knowledge only.
#[derive(Debug)]
pub enum RemoteSendError<M> {
/// No live connection to that node right now (never connected, or gone
/// and not yet re-dialed). The message is handed back.
NotConnected(M),
/// The payload did not serialize.
Encode(M, PayloadError),
}
impl<M> RemoteSendError<M> {
pub fn into_inner(self) -> M {
match self {
RemoteSendError::NotConnected(m) | RemoteSendError::Encode(m, _) => m,
}
}
}
/// The outbound table, one per runtime (a `RuntimeInner` field): per live
/// node, its current incarnation (the RFC v2 §3 send-site check) and the
/// connection's dedicated outbound sender. Plus this node's own wire
/// identity, which pid serialization stamps.
pub(crate) struct Outbound {
by_node: HashMap<String, Route>,
local: Option<(String, Incarnation)>,
}
/// One live connection as the outbound path sees it: the peer's current
/// incarnation and the two inboxes of its connection actor — frames (c9)
/// and monitor bookkeeping (c12, [`MonCmd`]).
pub(crate) struct Route {
incarnation: Incarnation,
frames: Sender<Frame>,
monitors: Sender<MonCmd>,
}
impl Outbound {
pub(crate) fn new() -> Self {
Outbound {
by_node: HashMap::new(),
local: None,
}
}
}
/// Set this node's wire identity — what serialized pids are stamped with
/// and what a `RemotePid` must name to collapse. `cluster::start` sets it;
/// exposed for local tests. Must run inside [`run`](crate::run).
pub fn set_local_identity(node: &str, incarnation: Incarnation) {
with_runtime(|inner| {
inner.outbound.lock().local = Some((node.to_string(), incarnation));
});
}
/// This node's wire identity, if set. Must run inside [`run`](crate::run).
pub fn local_identity() -> Option<(String, Incarnation)> {
with_runtime(|inner| inner.outbound.lock().local.clone())
}
/// Manager-only: bind `node`'s outbound channels at `incarnation`. Called
/// inside `Register`.
pub(crate) fn bind_outbound(
node: &str,
incarnation: Incarnation,
frames: Sender<Frame>,
monitors: Sender<MonCmd>,
) {
with_runtime(|inner| {
inner.outbound.lock().by_node.insert(
node.to_string(),
Route {
incarnation,
frames,
monitors,
},
);
});
}
/// Test probe: bind an arbitrary sender as `node`'s outbound so a test can
/// assert what frames leave — or don't. Same table, same lookup as the real
/// path (this is how "no frame emitted" is asserted at the frame level).
/// Frames only: there is no connection actor behind a probe, so a
/// [`monitor_remote`] against a probed node reports `Disconnected`.
pub fn bind_outbound_probe(node: &str, incarnation: Incarnation, tx: Sender<Frame>) {
drop(bind_outbound_probe_with_monitors(node, incarnation, tx));
}
/// The monitor half of a probed node's inbox: opaque, held only to be
/// dropped. See [`bind_outbound_probe_with_monitors`].
pub struct MonitorInbox {
_rx: Receiver<MonCmd>,
}
/// Test probe: like [`bind_outbound_probe`], but the monitor-command
/// receiver is handed back instead of dropped, so a test can stage the
/// c13 drain gap — a `Monitor` command that reached the connection's inbox
/// and dies unread when the inbox is dropped. While the inbox lives,
/// [`monitor_remote`] against the probed node is simply in flight.
pub fn bind_outbound_probe_with_monitors(
node: &str,
incarnation: Incarnation,
tx: Sender<Frame>,
) -> MonitorInbox {
let (mon_tx, mon_rx) = channel();
bind_outbound(node, incarnation, tx, mon_tx);
MonitorInbox { _rx: mon_rx }
}
/// Manager-only: unbind `node`'s outbound channel. Called on `Disconnect`,
/// reap, and manager shutdown. Dropping the sender is what closes the conn
/// actor's outbound arm — but that arm's closure is NOT a stop signal (the
/// cmd channel is, per D9); the actor simply stops selecting on it.
pub(crate) fn unbind_outbound(node: &str) {
with_runtime(|inner| {
inner.outbound.lock().by_node.remove(node);
});
}
/// Send `msg` to `target`. `Ok(())` = handed to the connection's inbox, and
/// nothing more — see the module docs. Must run inside
/// [`run`](crate::run).
pub fn send<M>(target: RemoteName<M>, msg: M) -> Result<(), RemoteSendError<M>>
where
M: serde::Serialize + Send + 'static,
{
let payload = match encode_payload(&msg) {
Ok(p) => p,
Err(e) => return Err(RemoteSendError::Encode(msg, e)),
};
let frame = Frame::SendNamed {
name: target.name.as_str().to_string(),
type_hash: type_hash::<M>(),
payload,
};
match hand_to_connection(&target.node, frame) {
Ok(()) => Ok(()),
Err(NotConnected) => Err(RemoteSendError::NotConnected(msg)),
}
}
/// No live connection to the named node — the payload-free form of
/// [`RemoteSendError::NotConnected`], for the raw path.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NotConnected;
/// The untyped escape hatch: send pre-encoded `payload` under an explicit
/// `type_hash`. Exists so tests (and future codecs) can put deliberately
/// wrong frames on the wire; the typed [`send`] cannot express a hash/type
/// mismatch, by design. Same `Ok` semantics as [`send`].
pub fn send_remote_raw(
node: &str,
name: &str,
type_hash: u64,
payload: &[u8],
) -> Result<(), NotConnected> {
hand_to_connection(
node,
Frame::SendNamed {
name: name.to_string(),
type_hash,
payload: payload.to_vec(),
},
)
}
/// One lookup, one send. Clone the sender out under the lock and send
/// outside it (a channel send can unpark the conn actor).
fn hand_to_connection(node: &str, frame: Frame) -> Result<(), NotConnected> {
let tx = with_runtime(|inner| {
inner
.outbound
.lock()
.by_node
.get(node)
.map(|r| r.frames.clone())
});
match tx {
Some(tx) => tx.send(frame).map_err(|_| NotConnected),
None => Err(NotConnected),
}
}
// ---- pids ---------------------------------------------------------------
/// A pid on some node: `(node_name, incarnation, index, generation)` plus
/// the actor type. See the module docs. Serializes as a 4-tuple.
pub struct RemotePid<A> {
node: String,
incarnation: Incarnation,
index: u32,
generation: u32,
_marker: PhantomData<fn() -> A>,
}
impl<A> RemotePid<A> {
/// Build from raw parts (tests, and codecs re-hydrating a pid).
pub fn from_parts(
node: impl Into<String>,
incarnation: Incarnation,
index: u32,
generation: u32,
) -> Self {
RemotePid {
node: node.into(),
incarnation,
index,
generation,
_marker: PhantomData,
}
}
/// The wire form of a local pid, stamped with this node's identity, and
/// **marked watchable** — asking for the wire form *is* the intent to
/// ship the pid, so this is the same D12 set-site as `Pid::serialize`
/// (c12 made it explicit: a peer may monitor exactly the pids that
/// crossed, and a pid handed out via `from_local` in a hand-built reply
/// has crossed). Must run inside [`run`](crate::run).
///
/// `None` when this runtime has no wire identity (no `cluster::start`,
/// no [`set_local_identity`]): such a pid cannot name a node, and a
/// stamped `("", 0)` would be dropped by every peer with no signal.
/// The pid is not marked watchable in that case either.
pub fn from_local(pid: Pid<A>) -> Option<Self> {
let (node, incarnation) = local_identity()?;
Some(Self::from_local_at(pid, node, incarnation))
}
/// `from_local` with the identity supplied by the caller — for a holder
/// that already carries the node's identity (the pg actor) and must not
/// have a `None` path. Marks watchable like `from_local`.
pub(crate) fn from_local_at(pid: Pid<A>, node: String, incarnation: Incarnation) -> Self {
crate::monitor::mark_watchable(pid);
RemotePid::from_parts(node, incarnation, pid.index(), pid.generation())
}
/// The collapse: `Some(local pid)` iff this pid names this very node
/// (name and incarnation). Must run inside [`run`](crate::run).
pub fn local(&self) -> Option<Pid<A>> {
let (n, i) = local_identity()?;
(n == self.node && i == self.incarnation)
.then(|| crate::pid::assert_type::<A>(Pid::new(self.index, self.generation)))
}
/// Drop the actor type: the untyped `RemotePid<Erased>`, the form
/// [`RemoteDown`] and [`RemoteMonitor`] carry (mirrors [`Pid::erase`]).
pub fn erase(self) -> RemotePid<Erased> {
RemotePid::from_parts(self.node, self.incarnation, self.index, self.generation)
}
/// Re-type an erased pid as `RemotePid<B>` — the unchecked mirror of
/// `pid::assert_type`, with the same degradation: a wrong `B` means the
/// target refuses the payload's hash (never a misroute).
pub(crate) fn assert_type<B>(self) -> RemotePid<B> {
RemotePid::from_parts(self.node, self.incarnation, self.index, self.generation)
}
pub fn node(&self) -> &str {
&self.node
}
pub fn incarnation(&self) -> Incarnation {
self.incarnation
}
pub fn index(&self) -> u32 {
self.index
}
pub fn generation(&self) -> u32 {
self.generation
}
}
impl<A> Clone for RemotePid<A> {
fn clone(&self) -> Self {
RemotePid::from_parts(
self.node.clone(),
self.incarnation,
self.index,
self.generation,
)
}
}
impl<A> PartialEq for RemotePid<A> {
fn eq(&self, o: &Self) -> bool {
self.node == o.node
&& self.incarnation == o.incarnation
&& self.index == o.index
&& self.generation == o.generation
}
}
impl<A> Eq for RemotePid<A> {}
impl<A> std::fmt::Debug for RemotePid<A> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"<{}.{}@{}#{}>",
self.index,
self.generation,
self.node,
self.incarnation.get()
)
}
}
impl<A> serde::Serialize for RemotePid<A> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
(
self.node.as_str(),
self.incarnation.get(),
self.index,
self.generation,
)
.serialize(s)
}
}
impl<'de, A> serde::Deserialize<'de> for RemotePid<A> {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (node, inc, index, generation) = <(String, u32, u32, u32)>::deserialize(d)?;
Ok(RemotePid::from_parts(
node,
Incarnation::new(inc),
index,
generation,
))
}
}
/// Why a pid-targeted send did not leave this node. Local knowledge only.
#[derive(Debug)]
pub enum ToRemoteError<M> {
/// No live connection to the pid's node.
NotConnected(M),
/// The pid's incarnation is not that node's current one (RFC v2 §3): the
/// actor died with its incarnation. Detected at the send site; no frame.
DeadIncarnation(M),
/// The payload did not serialize.
Encode(M, PayloadError),
/// The pid collapsed to a local one and the local typed send failed.
Local(SendError<M>),
}
impl<M> ToRemoteError<M> {
/// The undelivered message.
pub fn into_inner(self) -> M {
match self {
ToRemoteError::NotConnected(m)
| ToRemoteError::DeadIncarnation(m)
| ToRemoteError::Encode(m, _) => m,
ToRemoteError::Local(e) => e.into_inner(),
}
}
}
/// Send `msg` to a pid, wherever it lives. Self-node pids short-circuit to
/// the local typed send with `msg` itself (no encode, no frame); others go
/// out as a `Send` frame after the incarnation check. `Ok(())` for a remote
/// target = handed to the connection's inbox. Must run inside
/// [`run`](crate::run).
pub fn send_to_remote<A>(target: RemotePid<A>, msg: A::Msg) -> Result<(), ToRemoteError<A::Msg>>
where
A: Addressable,
A::Msg: serde::Serialize,
{
if let Some(local) = target.local() {
return send_to(local, msg).map_err(ToRemoteError::Local);
}
let route = with_runtime(|inner| {
inner
.outbound
.lock()
.by_node
.get(&target.node)
.map(|r| (r.incarnation, r.frames.clone()))
});
let (current, tx) = match route {
Some(r) => r,
None => return Err(ToRemoteError::NotConnected(msg)),
};
if current != target.incarnation {
return Err(ToRemoteError::DeadIncarnation(msg));
}
let payload = match encode_payload(&msg) {
Ok(p) => p,
Err(e) => return Err(ToRemoteError::Encode(msg, e)),
};
let frame = Frame::Send {
index: target.index,
generation: target.generation,
type_hash: type_hash::<A::Msg>(),
payload,
};
tx.send(frame).map_err(|_| ToRemoteError::NotConnected(msg))
}
/// The inbound `Send` seam: deliver `payload` under `type_hash` to the local
/// actor `(index, generation)`. Node and incarnation are implicit in the
/// connection (bound at handshake) — the frame carries only the slot
/// identity. Delivery goes through c8's decoder table, so only types the
/// receiver has [`expose_type`](crate::cluster::expose::expose_type)d (or
/// exposed by name) can land; anything else is refused, never misrouted.
pub fn deliver_to_pid(
index: u32,
generation: u32,
type_hash: u64,
payload: &[u8],
) -> InboundVerdict {
let pid = Pid::new(index, generation);
match decode_deliver(type_hash, pid, payload) {
Ok(()) => InboundVerdict::Delivered,
Err(e) => InboundVerdict::Refused(e),
}
}
/// What the inbound seam did with a `SendNamed`. Local observability only;
/// nothing goes back on the wire (RFC §3).
#[derive(Debug)]
pub enum InboundVerdict {
/// Decoded and handed to the name's holder.
Delivered,
/// The name is not in this node's exposed set.
NotExposed,
/// The frame's hash is not the hash the name was exposed with.
HashMismatch { expected: u64, got: u64 },
/// Exposed, but no live holder right now (unbound, or its holder died
/// and the binding is being pruned).
Unresolved,
/// Resolved, but the delivery half refused it (decode failure, or the
/// holder's channel does not accept the exposed type — a local
/// re-registration under a different type; never a misroute).
Refused(DeliverError),
}
impl InboundVerdict {
/// A short static label for tracing/counting (`smarm-trace` records one
/// `ClusterInbound` event per frame with it).
pub fn label(&self) -> &'static str {
match self {
InboundVerdict::Delivered => "delivered",
InboundVerdict::NotExposed => "not_exposed",
InboundVerdict::HashMismatch { .. } => "hash_mismatch",
InboundVerdict::Unresolved => "unresolved",
InboundVerdict::Refused(_) => "refused",
}
}
}
/// THE inbound resolution seam: exposed-set check → hash check → registry
/// resolution → c8 delivery. See the module docs. Must run inside
/// [`run`](crate::run) — the conn actor's context.
pub fn deliver_named(name: &str, type_hash: u64, payload: &[u8]) -> InboundVerdict {
let Some(expected) = exposed_hash(name) else {
return InboundVerdict::NotExposed;
};
if expected != type_hash {
return InboundVerdict::HashMismatch {
expected,
got: type_hash,
};
}
let Some(pid) = whereis(name) else {
return InboundVerdict::Unresolved;
};
match decode_deliver(type_hash, pid, payload) {
Ok(()) => InboundVerdict::Delivered,
Err(e) => InboundVerdict::Refused(e),
}
}
// ---- monitors (c12) -----------------------------------------------------
/// Bookkeeping commands from [`monitor_remote`]/[`demonitor_remote`] to the
/// connection actor that owns the link to the target's node. The actor
/// records the registration and *then* emits the `Monitor` frame itself, so
/// a `Down` can never arrive at a table that does not yet know the id. It
/// lives in the actor (not on `RuntimeInner`) so the bookkeeping dies with
/// the connection — exactly what c13 needs to synthesize `Disconnected`.
pub(crate) enum MonCmd {
Monitor {
id: MonitorId,
target: RemotePid<Erased>,
tx: Sender<RemoteDown>,
},
Demonitor {
id: MonitorId,
},
}
/// A remotely-monitored actor's termination notice — the cluster analog of
/// [`Down`](crate::monitor::Down), with the pid in its wire form because it
/// may name any node.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RemoteDown {
/// The pid that was being monitored.
pub pid: RemotePid<Erased>,
/// How it went down. `Disconnected` means the *link* to its node was
/// lost (or absent) — nothing is known about the actor itself.
pub reason: RemoteDownReason,
}
enum Watch {
/// The target collapsed to this node: an ordinary local monitor,
/// translated on read.
Local(Monitor),
/// The target is elsewhere: the connection actor for its node holds the
/// registration and forwards the peer's `Down` frame here. The
/// `RemoteState` is the read-side backstop (c13): a channel that closes
/// while `Live` — the connection died with our command unread — reads
/// as `Disconnected` once; afterwards, and after a cancel, closed is
/// just closed.
Remote(Receiver<RemoteDown>, Cell<RemoteState>),
}
/// Where a remote-watch stands from the reader's side.
#[derive(Clone, Copy, PartialEq, Eq)]
enum RemoteState {
/// No notice yet, not cancelled: a closed channel means `Disconnected`.
Live,
/// The one notice has been read (or synthesized): nothing more is due.
Done,
/// `demonitor_remote` ran: never synthesize.
Cancelled,
}
/// A live remote monitor: read its one [`RemoteDown`] with
/// [`recv`](RemoteMonitor::recv)/[`try_recv`](RemoteMonitor::try_recv), or
/// fold it into a `select` via [`arm`](RemoteMonitor::arm). Distinct from
/// [`Monitor`] on purpose: its target is a [`RemotePid`], its notice a
/// [`RemoteDown`], and it can report `Disconnected` — none of which a local
/// monitor can express. Dropping it discards an unread notice, like the
/// local one; after [`demonitor_remote`] the channel is closed and empty, so
/// `recv` errs rather than parking — also like the local one.
///
/// Exactly one notice is guaranteed even if the connection actor dies with
/// the registration unread (the c13 drain gap): a channel that closes
/// before any notice — and before any cancel — reads as `Disconnected`,
/// once. The next read is the ordinary closed-channel `Err`.
pub struct RemoteMonitor {
/// This registration's process-unique id — minted here, echoed by the
/// peer in its `Down` frame.
pub id: MonitorId,
/// The pid being monitored.
pub target: RemotePid<Erased>,
watch: Watch,
}
impl RemoteMonitor {
/// Block (cooperatively) for the notice.
pub fn recv(&self) -> Result<RemoteDown, RecvError> {
match &self.watch {
Watch::Local(m) => m.rx.recv().map(|d| RemoteDown {
pid: self.target.clone(),
reason: d.reason.into(),
}),
Watch::Remote(rx, st) => match rx.recv() {
Ok(d) => {
st.set(RemoteState::Done);
Ok(d)
}
Err(e) => self.closed(st).ok_or(e),
},
}
}
/// The notice if it has arrived; `Ok(None)` if not yet.
pub fn try_recv(&self) -> Result<Option<RemoteDown>, RecvError> {
match &self.watch {
Watch::Local(m) => m.rx.try_recv().map(|o| {
o.map(|d| RemoteDown {
pid: self.target.clone(),
reason: d.reason.into(),
})
}),
Watch::Remote(rx, st) => match rx.try_recv() {
Ok(Some(d)) => {
st.set(RemoteState::Done);
Ok(Some(d))
}
Ok(None) => Ok(None),
Err(e) => self.closed(st).map(Some).ok_or(e),
},
}
}
/// The channel closed. While `Live` — no notice yet, no cancel — that
/// is the connection having died with our registration unread, so
/// synthesize the one `Disconnected` and mark `Done`; otherwise closed
/// is just closed.
fn closed(&self, st: &Cell<RemoteState>) -> Option<RemoteDown> {
if st.get() != RemoteState::Live {
return None;
}
st.set(RemoteState::Done);
Some(RemoteDown {
pid: self.target.clone(),
reason: RemoteDownReason::Disconnected,
})
}
/// The selectable arm: readiness means [`try_recv`](Self::try_recv)
/// will yield the notice.
pub fn arm(&self) -> &dyn Selectable {
match &self.watch {
Watch::Local(m) => &m.rx,
Watch::Remote(rx, _) => rx,
}
}
}
impl std::fmt::Debug for RemoteMonitor {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RemoteMonitor")
.field("id", &self.id)
.field("target", &self.target)
.finish_non_exhaustive()
}
}
/// Monitor `target`, wherever it lives. Exactly one [`RemoteDown`] arrives:
///
/// - self-node pid ⇒ an ordinary local monitor underneath (same NoProc rule);
/// - no live connection to the pid's node ⇒ `Disconnected`, queued at once
/// (the remote analog of NoProc: nothing can be known);
/// - the pid's incarnation is not the node's current one ⇒ `NoProc`, queued
/// at once — the node is *known* to have restarted, so its actor is a
/// corpse, not a partition (RFC v2 §3);
/// - otherwise the connection actor registers the id and sends `Monitor`;
/// the peer answers with the true terminal reason on exit, or immediately
/// with the recorded reason for a corpse (`terminal_reason`, RFC §6) or
/// `NoProc` for a pid it never exposed and never shipped.
///
/// The connection dropping while the monitor is outstanding delivers
/// `Disconnected` (c13): the connection actor synthesizes it on teardown,
/// and the monitor's own read path backstops the case where the actor died
/// with the registration still unread. Must run inside [`run`](crate::run).
pub fn monitor_remote<A>(target: RemotePid<A>) -> RemoteMonitor {
if let Some(local) = target.local() {
let m = monitor(local);
return RemoteMonitor {
id: m.id,
target: target.erase(),
watch: Watch::Local(m),
};
}
let target = target.erase();
let (id, route) = with_runtime(|inner| {
let id = inner.alloc_monitor_id();
let route = inner
.outbound
.lock()
.by_node
.get(&target.node)
.map(|r| (r.incarnation, r.monitors.clone()));
(id, route)
});
let (tx, rx) = channel::<RemoteDown>();
let immediate = match route {
None => Some(RemoteDownReason::Disconnected),
Some((current, _)) if current != target.incarnation => {
Some(RemoteDownReason::Local(crate::monitor::DownReason::NoProc))
}
Some((_, mon_tx)) => {
let cmd = MonCmd::Monitor {
id,
target: target.clone(),
tx: tx.clone(),
};
match mon_tx.send(cmd) {
Ok(()) => None,
Err(_) => Some(RemoteDownReason::Disconnected), // actor already gone
}
}
};
if let Some(reason) = immediate {
let _ = tx.send(RemoteDown {
pid: target.clone(),
reason,
});
}
RemoteMonitor {
id,
target,
watch: Watch::Remote(rx, Cell::new(RemoteState::Live)),
}
}
/// Cancel `m`. No future notice will be *sent* for it; a notice already in
/// flight from the peer is dropped on arrival, and one already sitting in
/// `m` is discarded when `m` is dropped (same contract as
/// [`demonitor`]). Unlike the local form this returns nothing: the
/// registration is owned by the connection actor, so whether the `Down`
/// beat the cancel is not local knowledge. Must run inside
/// [`run`](crate::run).
pub fn demonitor_remote(m: &RemoteMonitor) {
match &m.watch {
Watch::Local(local) => {
let _ = demonitor(local);
}
Watch::Remote(_, st) => {
// Cancel first: a channel closing after this is closed, not a
// Disconnected notice — the caller asked for silence.
st.set(RemoteState::Cancelled);
let mon_tx = with_runtime(|inner| {
inner
.outbound
.lock()
.by_node
.get(&m.target.node)
.map(|r| r.monitors.clone())
});
if let Some(mon_tx) = mon_tx {
let _ = mon_tx.send(MonCmd::Demonitor { id: m.id });
}
}
}
}
+298
View File
@@ -0,0 +1,298 @@
//! RFC 010 c3 — transport abstraction for the **control** connection.
//!
//! Scope, per RFC 010 v2 §5 and D2:
//!
//! - A "connection" here is the *control* connection: the one carrying this
//! RFC's frame inventory ([`crate::cluster::envelope::Frame`]), whose
//! heartbeats feed failure detection. The trait deliberately says nothing
//! about how many connections a peer pair may hold — the jarred rkyv bulk
//! plane opens **additional per-peer connections** outside this trait, and
//! nothing here may foreclose that.
//! - Homogeneous smarm⇄smarm only. The BEAM membrane is *not* a transport
//! impl and the trait does not accommodate it (D2).
//! - Addresses are opaque, **pre-resolved** strings. Name resolution is a
//! single separate seam (roadmap c9); impls reject unresolved names rather
//! than resolving them.
//!
//! Blocking model: [`Conn`] calls block the caller. The TCP impl parks the
//! calling *actor* (fd readiness via the scheduler); the loopback impl blocks
//! the calling *OS thread* and is a test transport — do not drive it from a
//! scheduler thread.
//!
//! Framing is not part of the trait: [`FramedConn`] is the single shared
//! codec that turns any byte-stream [`Conn`] into a frame pipe, feeding
//! [`Frame::decode`]'s incremental contract. Impls never re-implement
//! framing, and the conformance suite exercises the same codec over every
//! impl.
use std::io;
use crate::cluster::envelope::{DecodeError, EncodeError, Frame};
pub mod loopback;
pub mod tcp;
/// An established control connection: a bidirectional byte stream.
pub trait Conn: Send {
/// Read at least one byte, blocking the caller until data is available,
/// EOF, or error. `Ok(0)` means EOF: the peer closed and all bytes it
/// wrote before closing have been consumed.
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize>;
/// Write the whole buffer, blocking the caller as needed.
fn write_all(&mut self, buf: &[u8]) -> io::Result<()>;
/// Close both directions. Idempotent. Bytes already written remain
/// readable at the peer, which then observes EOF; peer writes after this
/// fail.
fn close(&mut self);
/// Diagnostic label for logs only. Mesh identity comes from the
/// handshake (`Hello`/`HelloAck`), never from the transport.
fn peer_addr(&self) -> String;
/// Readiness as a [`select`](crate::select) arm, for transports backed by
/// a file descriptor. `Some` lets a driver wait on "this connection is
/// readable" alongside an ordinary command inbox in a single `select`, so
/// one actor can interleave reading with control messages without a
/// second thread. The default is `None`: a transport with no fd (the
/// in-memory loopback) cannot be selected on and must be driven another
/// way.
fn readable_arm(&self) -> Option<crate::scheduler::FdArm> {
None
}
}
/// A bound listen point producing inbound [`Conn`]s.
pub trait Listener: Send {
/// Accept the next inbound connection, blocking the caller.
fn accept(&mut self) -> io::Result<Box<dyn Conn>>;
/// The concrete bound address, dialable as-is (e.g. the real port when
/// bound with port 0).
fn local_addr(&self) -> String;
/// Readiness as a [`select`](crate::select) arm, mirroring
/// [`Conn::readable_arm`]: `Some` lets an acceptor wait on "an inbound
/// connection is pending" alongside a command inbox in one `select`, so
/// it can be told to stop without a poll loop. Default `None` (the
/// loopback listener has no fd and must be driven synchronously).
fn readable_arm(&self) -> Option<crate::scheduler::FdArm> {
None
}
}
/// A way of establishing control connections. Object-safe on purpose: the
/// connector and membership layers hold `&dyn Transport` / boxed conns
/// rather than growing a generic parameter.
pub trait Transport: Send + Sync {
/// Connect to a peer's listen address. Blocks the caller until
/// established or failed.
fn dial(&self, addr: &str) -> io::Result<Box<dyn Conn>>;
/// Bind a listen point.
fn listen(&self, addr: &str) -> io::Result<Box<dyn Listener>>;
}
impl std::fmt::Debug for dyn Conn {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Conn({})", self.peer_addr())
}
}
impl std::fmt::Debug for dyn Listener {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Listener({})", self.local_addr())
}
}
/// Error surface of [`FramedConn::send`].
#[derive(Debug)]
pub enum SendError {
/// The frame could not be encoded (e.g. a field over its wire limit).
Encode(EncodeError),
/// The transport failed mid-write.
Io(io::Error),
}
impl std::fmt::Display for SendError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SendError::Encode(e) => write!(f, "frame encode failed: {e:?}"),
SendError::Io(e) => write!(f, "transport write failed: {e}"),
}
}
}
impl std::error::Error for SendError {}
/// Error surface of [`FramedConn::recv`].
#[derive(Debug)]
pub enum RecvError {
/// The byte stream is not a valid frame stream (bad tag, lying length,
/// oversized frame, …). The connection is unusable.
Corrupt(DecodeError),
/// The peer closed mid-frame: EOF arrived with a partial frame buffered.
/// Distinct from a clean close, which is `Ok(None)`.
TruncatedByPeer,
/// The transport failed mid-read.
Io(io::Error),
/// The deadline passed before a full frame arrived
/// ([`FramedConn::recv_deadline`] only; plain [`recv`](FramedConn::recv)
/// never returns this).
TimedOut,
}
impl std::fmt::Display for RecvError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RecvError::Corrupt(e) => write!(f, "frame stream corrupt: {e:?}"),
RecvError::TruncatedByPeer => write!(f, "peer closed mid-frame"),
RecvError::Io(e) => write!(f, "transport read failed: {e}"),
RecvError::TimedOut => write!(f, "deadline passed mid-receive"),
}
}
}
impl std::error::Error for RecvError {}
/// How many bytes each blocking read asks the transport for.
const READ_CHUNK: usize = 8 * 1024;
/// The shared framed codec: one of these per control connection, owning the
/// [`Conn`] and the reassembly buffer. Frames may arrive split or coalesced
/// arbitrarily; [`recv`](FramedConn::recv) reassembles either way.
pub struct FramedConn {
conn: Box<dyn Conn>,
rbuf: Vec<u8>,
}
impl FramedConn {
pub fn new(conn: Box<dyn Conn>) -> Self {
FramedConn {
conn,
rbuf: Vec::new(),
}
}
/// Encode and write one frame.
pub fn send(&mut self, frame: &Frame) -> Result<(), SendError> {
let mut out = Vec::new();
frame.encode(&mut out).map_err(SendError::Encode)?;
self.conn.write_all(&out).map_err(SendError::Io)
}
/// Receive the next frame. `Ok(None)` is a clean close: EOF at a frame
/// boundary. EOF mid-frame is [`RecvError::TruncatedByPeer`].
pub fn recv(&mut self) -> Result<Option<Frame>, RecvError> {
loop {
match Frame::decode(&self.rbuf) {
Ok(Some((frame, consumed))) => {
self.rbuf.drain(..consumed);
return Ok(Some(frame));
}
Ok(None) => {}
Err(e) => return Err(RecvError::Corrupt(e)),
}
let mut chunk = [0u8; READ_CHUNK];
let n = self.conn.read(&mut chunk).map_err(RecvError::Io)?;
if n == 0 {
return if self.rbuf.is_empty() {
Ok(None)
} else {
Err(RecvError::TruncatedByPeer)
};
}
self.rbuf.extend_from_slice(&chunk[..n]);
}
}
/// Like [`recv`](FramedConn::recv), but gives up with
/// [`RecvError::TimedOut`] once `deadline` passes without a full frame.
/// The deadline is enforced between reads via the connection's fd arm
/// (so the caller must be an actor); a transport with no fd (loopback)
/// cannot be timed out and this degrades to a plain blocking `recv` —
/// the same caveat as liveness.
pub fn recv_deadline(
&mut self,
deadline: std::time::Instant,
) -> Result<Option<Frame>, RecvError> {
loop {
match Frame::decode(&self.rbuf) {
Ok(Some((frame, consumed))) => {
self.rbuf.drain(..consumed);
return Ok(Some(frame));
}
Ok(None) => {}
Err(e) => return Err(RecvError::Corrupt(e)),
}
if let Some(arm) = self.conn.readable_arm() {
let left = deadline.saturating_duration_since(std::time::Instant::now());
if left.is_zero() {
return Err(RecvError::TimedOut);
}
match crate::channel::try_select_timeout(&[&arm], left) {
Ok(Some(_)) => {}
Ok(None) => return Err(RecvError::TimedOut),
Err(e) => return Err(RecvError::Io(e)),
}
}
let mut chunk = [0u8; READ_CHUNK];
let n = self.conn.read(&mut chunk).map_err(RecvError::Io)?;
if n == 0 {
return if self.rbuf.is_empty() {
Ok(None)
} else {
Err(RecvError::TruncatedByPeer)
};
}
self.rbuf.extend_from_slice(&chunk[..n]);
}
}
/// One socket read, appended to the reassembly buffer. Returns the byte
/// count (`0` = EOF). For select-loop callers that were just told the fd
/// is readable: under the level-triggered IO thread exactly one read per
/// readable wake never blocks and never loses data — leftover socket
/// bytes re-signal on the next select, and complete frames already
/// reassembled are drained with [`next_buffered`](FramedConn::next_buffered).
/// (A plain [`recv`](FramedConn::recv) can block into the socket while
/// the buffer holds a partial frame, which a loop with deadlines to keep
/// cannot afford.)
pub fn read_once(&mut self) -> std::io::Result<usize> {
let mut chunk = [0u8; READ_CHUNK];
let n = self.conn.read(&mut chunk)?;
self.rbuf.extend_from_slice(&chunk[..n]);
Ok(n)
}
/// Decode the next complete frame already sitting in the reassembly
/// buffer, without touching the socket. `Ok(None)` means the buffer
/// holds no complete frame (empty, or a partial awaiting more bytes).
pub fn next_buffered(&mut self) -> Result<Option<Frame>, DecodeError> {
match Frame::decode(&self.rbuf) {
Ok(Some((frame, consumed))) => {
self.rbuf.drain(..consumed);
Ok(Some(frame))
}
Ok(None) => Ok(None),
Err(e) => Err(e),
}
}
/// Close the underlying connection (idempotent, see [`Conn::close`]).
pub fn close(&mut self) {
self.conn.close();
}
/// Diagnostic label of the underlying connection.
pub fn peer_addr(&self) -> String {
self.conn.peer_addr()
}
/// The underlying connection's readiness arm, if it is fd-backed (see
/// [`Conn::readable_arm`]).
pub fn readable_arm(&self) -> Option<crate::scheduler::FdArm> {
self.conn.readable_arm()
}
}
+274
View File
@@ -0,0 +1,274 @@
//! In-memory loopback transport — a shipped **test** transport.
//!
//! Lets Phases 2–4 exercise protocol logic (connector, membership,
//! monitors) through the real transport trait and the real framed codec
//! without sockets or timing flake.
//!
//! Blocking model: calls block the **OS thread** on a condvar. That is the
//! right shape for plain `#[test]`s driving protocol state machines; it is
//! the wrong shape for scheduler threads. Do not drive a loopback conn from
//! inside an actor — use the TCP impl there.
//!
//! Semantics mirror TCP shutdown where it matters for the codec: bytes
//! written before `close` remain readable at the peer, which then sees EOF;
//! writes toward a closed peer fail with `BrokenPipe`. Write buffers are
//! unbounded, so writes never block — backpressure is not simulated.
use std::collections::{HashMap, VecDeque};
use std::io;
use std::sync::{Arc, Condvar, Mutex, MutexGuard};
use super::{Conn, Listener, Transport};
/// Poison-tolerant lock: a panicked holder in a *test* transport must not
/// cascade; the byte-queue state stays consistent under every early return.
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
match m.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
}
}
// ---------------------------------------------------------------------------
// One direction of a duplex: a byte queue with close flags for both ends
// ---------------------------------------------------------------------------
#[derive(Default)]
struct PipeState {
bytes: VecDeque<u8>,
/// The writing end closed: readers drain remaining bytes, then EOF.
write_closed: bool,
/// The reading end closed: writers fail with `BrokenPipe`.
read_closed: bool,
}
#[derive(Default)]
struct Pipe {
state: Mutex<PipeState>,
cv: Condvar,
}
impl Pipe {
fn write_all(&self, buf: &[u8]) -> io::Result<()> {
let mut st = lock(&self.state);
if st.write_closed {
return Err(io::Error::new(
io::ErrorKind::NotConnected,
"loopback conn closed locally",
));
}
if st.read_closed {
return Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"loopback peer closed",
));
}
st.bytes.extend(buf);
self.cv.notify_all();
Ok(())
}
fn read(&self, buf: &mut [u8]) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
let mut st = lock(&self.state);
loop {
if !st.bytes.is_empty() {
let n = st.bytes.len().min(buf.len());
for (slot, byte) in buf.iter_mut().zip(st.bytes.drain(..n)) {
*slot = byte;
}
return Ok(n);
}
if st.write_closed || st.read_closed {
return Ok(0); // EOF: peer closed, or our own end closed.
}
st = match self.cv.wait(st) {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
}
}
/// Close from the writer side: remaining bytes stay readable, then EOF.
fn close_write(&self) {
lock(&self.state).write_closed = true;
self.cv.notify_all();
}
/// Close from the reader side: peer writes fail from now on.
fn close_read(&self) {
lock(&self.state).read_closed = true;
self.cv.notify_all();
}
}
// ---------------------------------------------------------------------------
// Conn: two pipes, one per direction
// ---------------------------------------------------------------------------
/// One end of an established loopback connection.
pub struct LoopbackConn {
tx: Arc<Pipe>,
rx: Arc<Pipe>,
peer: String,
}
impl Conn for LoopbackConn {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.rx.read(buf)
}
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
self.tx.write_all(buf)
}
fn close(&mut self) {
self.tx.close_write();
self.rx.close_read();
}
fn peer_addr(&self) -> String {
self.peer.clone()
}
}
impl Drop for LoopbackConn {
fn drop(&mut self) {
self.close();
}
}
fn conn_pair(listen_addr: &str, conn_no: u64) -> (LoopbackConn, LoopbackConn) {
let a_to_b = Arc::new(Pipe::default());
let b_to_a = Arc::new(Pipe::default());
let dialer = LoopbackConn {
tx: a_to_b.clone(),
rx: b_to_a.clone(),
peer: listen_addr.to_string(),
};
let accepted = LoopbackConn {
tx: b_to_a,
rx: a_to_b,
peer: format!("{listen_addr}#dialer-{conn_no}"),
};
(dialer, accepted)
}
// ---------------------------------------------------------------------------
// Listener + registry
// ---------------------------------------------------------------------------
#[derive(Default)]
struct AcceptState {
pending: VecDeque<LoopbackConn>,
closed: bool,
}
#[derive(Default)]
struct AcceptQueue {
state: Mutex<AcceptState>,
cv: Condvar,
}
/// A bound loopback listen point.
pub struct LoopbackListener {
addr: String,
queue: Arc<AcceptQueue>,
registry: Arc<Mutex<Registry>>,
}
impl Listener for LoopbackListener {
fn accept(&mut self) -> io::Result<Box<dyn Conn>> {
let mut st = lock(&self.queue.state);
loop {
if let Some(conn) = st.pending.pop_front() {
return Ok(Box::new(conn));
}
if st.closed {
return Err(io::Error::new(
io::ErrorKind::NotConnected,
"loopback listener closed",
));
}
st = match self.queue.cv.wait(st) {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
}
}
fn local_addr(&self) -> String {
self.addr.clone()
}
}
impl Drop for LoopbackListener {
fn drop(&mut self) {
lock(&self.registry).listeners.remove(&self.addr);
let mut st = lock(&self.queue.state);
st.closed = true;
self.queue.cv.notify_all();
}
}
#[derive(Default)]
struct Registry {
listeners: HashMap<String, Arc<AcceptQueue>>,
dial_count: u64,
}
/// The loopback transport. Addresses are arbitrary strings scoped to one
/// transport instance; distinct instances never see each other's listeners.
#[derive(Default)]
pub struct LoopbackTransport {
registry: Arc<Mutex<Registry>>,
}
impl Transport for LoopbackTransport {
fn dial(&self, addr: &str) -> io::Result<Box<dyn Conn>> {
let (queue, conn_no) = {
let mut reg = lock(&self.registry);
reg.dial_count += 1;
let no = reg.dial_count;
match reg.listeners.get(addr) {
Some(q) => (q.clone(), no),
None => {
return Err(io::Error::new(
io::ErrorKind::ConnectionRefused,
format!("no loopback listener at {addr:?}"),
));
}
}
};
let (dialer, accepted) = conn_pair(addr, conn_no);
let mut st = lock(&queue.state);
if st.closed {
return Err(io::Error::new(
io::ErrorKind::ConnectionRefused,
format!("loopback listener at {addr:?} closed"),
));
}
st.pending.push_back(accepted);
queue.cv.notify_all();
Ok(Box::new(dialer))
}
fn listen(&self, addr: &str) -> io::Result<Box<dyn Listener>> {
let queue = Arc::new(AcceptQueue::default());
let mut reg = lock(&self.registry);
if reg.listeners.contains_key(addr) {
return Err(io::Error::new(
io::ErrorKind::AddrInUse,
format!("loopback listener already bound at {addr:?}"),
));
}
reg.listeners.insert(addr.to_string(), queue.clone());
Ok(Box::new(LoopbackListener {
addr: addr.to_string(),
queue,
registry: self.registry.clone(),
}))
}
}
+285
View File
@@ -0,0 +1,285 @@
//! TCP transport — the production control-plane transport.
//!
//! Blocking model: every blocking point parks the **calling actor** on fd
//! readiness ([`crate::scheduler::wait_readable`] / `wait_writable`); the
//! scheduler thread is never blocked. All conn/listener methods must
//! therefore run inside an actor. `listen` itself only binds (no waiting)
//! and is callable anywhere.
//!
//! Addresses are pre-resolved `ip:port` strings (`SocketAddr` syntax, IPv4
//! or IPv6). Hostnames are rejected with `InvalidInput`: name resolution is
//! the single c9 seam, not something each transport does on the side.
//!
//! Writes use `send(2)` with `MSG_NOSIGNAL` — a peer reset must surface as
//! `BrokenPipe`/`ConnectionReset`, not `SIGPIPE`.
use std::io;
use std::net::{SocketAddr, TcpListener as StdListener, TcpStream};
use std::os::fd::{AsRawFd, RawFd};
use crate::scheduler::{wait_readable, wait_writable};
use super::{Conn, Listener, Transport};
// ---------------------------------------------------------------------------
// sockaddr plumbing
// ---------------------------------------------------------------------------
/// A `sockaddr_in`/`sockaddr_in6` built from a parsed `SocketAddr`, plus its
/// length, ready for `connect(2)`.
union SockAddrUnion {
v4: libc::sockaddr_in,
v6: libc::sockaddr_in6,
}
fn to_sockaddr(sa: &SocketAddr) -> (SockAddrUnion, libc::socklen_t) {
match sa {
SocketAddr::V4(v4) => {
let raw = libc::sockaddr_in {
sin_family: libc::AF_INET as libc::sa_family_t,
sin_port: v4.port().to_be(),
sin_addr: libc::in_addr {
s_addr: u32::from_be_bytes(v4.ip().octets()).to_be(),
},
sin_zero: [0; 8],
};
(
SockAddrUnion { v4: raw },
std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
)
}
SocketAddr::V6(v6) => {
let raw = libc::sockaddr_in6 {
sin6_family: libc::AF_INET6 as libc::sa_family_t,
sin6_port: v6.port().to_be(),
sin6_flowinfo: v6.flowinfo(),
sin6_addr: libc::in6_addr {
s6_addr: v6.ip().octets(),
},
sin6_scope_id: v6.scope_id(),
};
(
SockAddrUnion { v6: raw },
std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t,
)
}
}
}
fn parse_addr(addr: &str) -> io::Result<SocketAddr> {
addr.parse().map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("{addr:?} is not a resolved ip:port — resolution is the c9 seam"),
)
})
}
fn so_error(fd: RawFd) -> io::Result<()> {
let mut err: libc::c_int = 0;
let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
let rc = unsafe {
libc::getsockopt(
fd,
libc::SOL_SOCKET,
libc::SO_ERROR,
(&mut err) as *mut _ as *mut libc::c_void,
&mut len,
)
};
if rc != 0 {
return Err(io::Error::last_os_error());
}
if err != 0 {
return Err(io::Error::from_raw_os_error(err));
}
Ok(())
}
// ---------------------------------------------------------------------------
// Conn
// ---------------------------------------------------------------------------
/// One established TCP control connection. Owns the socket; drop closes it.
pub struct TcpConn {
stream: TcpStream,
closed: bool,
}
impl TcpConn {
fn fd(&self) -> RawFd {
self.stream.as_raw_fd()
}
}
impl Conn for TcpConn {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.closed {
return Ok(0);
}
if buf.is_empty() {
return Ok(0);
}
loop {
wait_readable(self.fd())?;
let n = unsafe { libc::read(self.fd(), buf.as_mut_ptr() as *mut _, buf.len()) };
if n >= 0 {
return Ok(n as usize);
}
let e = io::Error::last_os_error();
match e.kind() {
// Spurious readiness or signal: park again.
io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted => continue,
_ => return Err(e),
}
}
}
fn write_all(&mut self, mut buf: &[u8]) -> io::Result<()> {
if self.closed {
return Err(io::Error::new(
io::ErrorKind::NotConnected,
"tcp conn closed locally",
));
}
while !buf.is_empty() {
wait_writable(self.fd())?;
let n = unsafe {
libc::send(
self.fd(),
buf.as_ptr() as *const _,
buf.len(),
libc::MSG_NOSIGNAL,
)
};
if n >= 0 {
buf = &buf[n as usize..];
continue;
}
let e = io::Error::last_os_error();
match e.kind() {
io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted => continue,
_ => return Err(e),
}
}
Ok(())
}
fn close(&mut self) {
if !self.closed {
self.closed = true;
// Best-effort: the peer sees EOF after draining. The fd itself
// is released when the owning stream drops.
let _ = self.stream.shutdown(std::net::Shutdown::Both);
}
}
fn peer_addr(&self) -> String {
match self.stream.peer_addr() {
Ok(sa) => sa.to_string(),
Err(_) => "<disconnected>".to_string(),
}
}
fn readable_arm(&self) -> Option<crate::scheduler::FdArm> {
Some(crate::scheduler::FdArm::readable(self.fd()))
}
}
// ---------------------------------------------------------------------------
// Listener
// ---------------------------------------------------------------------------
/// A bound TCP listen point (non-blocking socket; accept parks the actor).
pub struct TcpListener {
inner: StdListener,
local: SocketAddr,
}
impl Listener for TcpListener {
fn readable_arm(&self) -> Option<crate::scheduler::FdArm> {
Some(crate::scheduler::FdArm::readable(self.inner.as_raw_fd()))
}
fn accept(&mut self) -> io::Result<Box<dyn Conn>> {
loop {
wait_readable(self.inner.as_raw_fd())?;
match self.inner.accept() {
Ok((stream, _peer)) => {
stream.set_nonblocking(true)?;
return Ok(Box::new(TcpConn {
stream,
closed: false,
}));
}
Err(e)
if e.kind() == io::ErrorKind::WouldBlock
|| e.kind() == io::ErrorKind::Interrupted =>
{
continue;
}
Err(e) => return Err(e),
}
}
}
fn local_addr(&self) -> String {
self.local.to_string()
}
}
// ---------------------------------------------------------------------------
// Transport
// ---------------------------------------------------------------------------
/// The TCP transport. Stateless; every call stands alone.
pub struct TcpTransport;
impl Transport for TcpTransport {
fn dial(&self, addr: &str) -> io::Result<Box<dyn Conn>> {
let sa = parse_addr(addr)?;
let family = match sa {
SocketAddr::V4(_) => libc::AF_INET,
SocketAddr::V6(_) => libc::AF_INET6,
};
let fd = unsafe {
libc::socket(
family,
libc::SOCK_STREAM | libc::SOCK_NONBLOCK | libc::SOCK_CLOEXEC,
0,
)
};
if fd < 0 {
return Err(io::Error::last_os_error());
}
// From here the fd is owned by `stream`; any early return drops it.
let stream = unsafe {
use std::os::fd::FromRawFd;
TcpStream::from_raw_fd(fd)
};
let (raw, len) = to_sockaddr(&sa);
let rc = unsafe { libc::connect(fd, (&raw) as *const _ as *const libc::sockaddr, len) };
if rc != 0 {
let e = io::Error::last_os_error();
if e.raw_os_error() != Some(libc::EINPROGRESS) {
return Err(e);
}
// Connect in flight: park until the socket is writable, then the
// verdict is in SO_ERROR.
wait_writable(fd)?;
so_error(fd)?;
}
Ok(Box::new(TcpConn {
stream,
closed: false,
}))
}
fn listen(&self, addr: &str) -> io::Result<Box<dyn Listener>> {
let sa = parse_addr(addr)?;
let inner = StdListener::bind(sa)?;
inner.set_nonblocking(true)?;
let local = inner.local_addr()?;
Ok(Box::new(TcpListener { inner, local }))
}
}
+2
View File
@@ -23,6 +23,8 @@
pub mod actor; pub mod actor;
pub mod causal; pub mod causal;
pub mod channel; pub mod channel;
#[cfg(feature = "cluster")]
pub mod cluster;
pub mod context; pub mod context;
pub mod gen_server; pub mod gen_server;
pub mod gen_statem; pub mod gen_statem;
+65 -35
View File
@@ -156,16 +156,32 @@ pub struct Monitor {
pub fn monitor<A>(target: Pid<A>) -> Monitor { pub fn monitor<A>(target: Pid<A>) -> Monitor {
let target = target.erase(); let target = target.erase();
let (tx, rx) = channel::<Down>(); let (tx, rx) = channel::<Down>();
let id = with_runtime(|inner| inner.alloc_monitor_id());
if !register_monitor(target, id, &tx) {
let _ = tx.send(Down {
pid: target,
reason: DownReason::NoProc,
});
}
Monitor { id, target, rx }
}
// Implementation note: registration happens under the target's cold /// Register a monitor `id` on `target` that delivers its `Down` to `tx` — the
// lock. `tx.clone()` takes the channel's own lock, a Channel-class /// primitive under [`monitor`], split out so a caller can fan many monitors
// RawMutex, which is explicitly permitted under a Leaf (cold) lock by /// into ONE channel (process groups: every membership's death lands on the
// the lock order documented in raw_mutex.rs. We must still not *send* /// reaper's single inbox). Returns `false` if `target` is already gone, in
// under the lock, since `Sender::send` can unpark a parked receiver, /// which case nothing is registered and the caller decides what to queue
// and there's no reason to nest that. /// (`monitor` sends `NoProc`). The caller allocates `id` up front so it can
let (id, registered) = with_runtime(|inner| { /// record the registration *before* arming it.
let id = inner.alloc_monitor_id(); ///
let registered = match inner.slot_at(target) { /// Implementation note: registration happens under the target's cold lock.
/// `tx.clone()` takes the channel's own lock, a Channel-class RawMutex, which
/// is explicitly permitted under a Leaf (cold) lock by the lock order
/// documented in raw_mutex.rs. We must still not *send* under the lock, since
/// `Sender::send` can unpark a parked receiver, and there's no reason to nest
/// that.
pub(crate) fn register_monitor(target: Pid, id: MonitorId, tx: &Sender<Down>) -> bool {
with_runtime(|inner| match inner.slot_at(target) {
Some(slot) => { Some(slot) => {
let mut cold = slot.cold.lock(); let mut cold = slot.cold.lock();
if slot.is_live_for(target) { if slot.is_live_for(target) {
@@ -176,18 +192,30 @@ pub fn monitor<A>(target: Pid<A>) -> Monitor {
} }
} }
None => false, None => false,
}; })
(id, registered) }
});
if !registered { /// Remove registration `id` from `target` — the primitive under
let _ = tx.send(Down { /// [`demonitor`], for callers that hold only the id (see
pid: target, /// [`register_monitor`]). `None` if the registration is not there: already
reason: DownReason::NoProc, /// fired, already removed, or the slot has moved on to a new tenant.
}); ///
/// The registration is removed under the target's cold lock, but the
/// `Sender` is moved *out* and dropped only after the lock is released.
/// Dropping the last sender runs `Sender::drop`, which may unpark a parked
/// receiver; legal under a cold lock, but pointless to nest.
pub(crate) fn unregister_monitor(target: Pid, id: MonitorId) -> Option<MonitorId> {
let removed: Option<(MonitorId, Sender<Down>)> = with_runtime(|inner| {
let slot = inner.slot_at(target)?;
let mut cold = slot.cold.lock();
if slot.generation() != target.generation() {
return None; // slot reused; the Down already fired
} }
let pos = cold.monitors.iter().position(|(mid, _)| *mid == id)?;
Monitor { id, target, rx } Some(cold.monitors.remove(pos))
});
// `removed`'s sender drops here, outside the lock.
removed.map(|(id, _sender)| id)
} }
/// Flag `target`'s tenancy as watchable: its death will stamp the slot's /// Flag `target`'s tenancy as watchable: its death will stamp the slot's
@@ -220,6 +248,23 @@ pub fn mark_watchable<A>(target: Pid<A>) {
}); });
} }
/// Whether `target` is live *and* its tenancy is watchable. The cluster's
/// remote-monitor admission check (RFC 010 c12): a peer may monitor a pid only
/// if that pid was exposed or crossed the wire (the D12 set-sites), and a
/// live-but-unwatchable pid answers exactly like a dead one — no liveness leak
/// beyond what `watchable` already grants. Same context contract as
/// [`monitor`].
#[cfg(feature = "cluster")]
pub(crate) fn is_watchable<A>(target: Pid<A>) -> bool {
let target = target.erase();
with_runtime(|inner| {
inner.slot_at(target).is_some_and(|slot| {
let cold = slot.cold.lock();
slot.is_live_for(target) && cold.watchable
})
})
}
/// The terminal [`DownReason`] of the tenancy `target` names, if that tenancy /// The terminal [`DownReason`] of the tenancy `target` names, if that tenancy
/// ever registered a name and is the *most recent named* death of its slot: /// ever registered a name and is the *most recent named* death of its slot:
/// finalize stamps the slot with `(generation, reason)` for once-registered /// finalize stamps the slot with `(generation, reason)` for once-registered
@@ -259,20 +304,5 @@ pub fn terminal_reason<A>(target: Pid<A>) -> Option<DownReason> {
/// instead of, or in addition to, calling this: dropping the [`Monitor`] /// instead of, or in addition to, calling this: dropping the [`Monitor`]
/// closes its receiver and any queued notice is discarded with it. /// closes its receiver and any queued notice is discarded with it.
pub fn demonitor(m: &Monitor) -> Option<MonitorId> { pub fn demonitor(m: &Monitor) -> Option<MonitorId> {
// Implementation note: the registration is removed under the target's unregister_monitor(m.target, m.id)
// cold lock, but the `Sender` is moved *out* and dropped only after the
// lock is released. Dropping the last sender runs `Sender::drop`, which
// may unpark a parked receiver; legal under a cold lock, but pointless
// to nest.
let removed: Option<(MonitorId, Sender<Down>)> = with_runtime(|inner| {
let slot = inner.slot_at(m.target)?;
let mut cold = slot.cold.lock();
if slot.generation() != m.target.generation() {
return None; // slot reused; the Down already fired
}
let pos = cold.monitors.iter().position(|(mid, _)| *mid == m.id)?;
Some(cold.monitors.remove(pos))
});
// `removed`'s sender drops here, outside the lock.
removed.map(|(id, _sender)| id)
} }
+463 -248
View File
@@ -99,10 +99,13 @@
//! ## Identity and clustering //! ## Identity and clustering
//! //!
//! A group member is described by a [`Member`] — a [`Pid`] plus a [`NodeId`] and //! A group member is described by a [`Member`] — a [`Pid`] plus a [`NodeId`] and
//! an [`Incarnation`]. Today everything is single-node, those two fields are //! an [`Incarnation`]. Everything on this page is **local**: you pass and
//! fixed defaults, and you only ever pass and receive a plain [`Pid`]: the extra //! receive plain [`Pid`]s, and [`members`] / [`pick`] / [`dispatch`] only ever
//! identity is carried so this API will not have to change when groups learn to //! name actors on this node (Erlang's `get_local_members`). With the `cluster`
//! span a cluster. //! feature a group also holds the members other nodes have announced, carried
//! under their [`NodeId`]; those never surface here — the cluster-wide reads
//! live in [`cluster::pg`](crate::cluster::pg) (`members_all` and friends) and
//! return a `Local | Remote` member type, since a [`Pid`] cannot hold a remote.
//! //!
//! ## Running context //! ## Running context
//! //!
@@ -110,10 +113,11 @@
//! from inside [`run`](crate::run) (that is, on an actor thread). Calling one //! from inside [`run`](crate::run) (that is, on an actor thread). Calling one
//! from outside a running runtime panics. //! from outside a running runtime panics.
use crate::monitor::{demonitor, monitor, Monitor}; use crate::channel::{channel, Sender};
use crate::monitor::{register_monitor, unregister_monitor, Down, DownReason, MonitorId};
use crate::pid::{assert_type, Addressable, Pid}; use crate::pid::{assert_type, Addressable, Pid};
use crate::registry::{send_to, SendError}; use crate::registry::{send_to, SendError};
use crate::scheduler::with_runtime; use crate::scheduler::{spawn_under, with_runtime};
use std::collections::HashMap; use std::collections::HashMap;
/// A cluster node handle. A `u32` integer handle, *not* an interned atom — the /// A cluster node handle. A `u32` integer handle, *not* an interned atom — the
@@ -186,13 +190,15 @@ pub struct Member {
pub pid: Pid, pub pid: Pid,
} }
/// One membership: a [`Member`] and the [`Monitor`] that watches its liveness. /// One membership: a [`Member`] and the id of the monitor that watches its
/// The monitor lives *alongside* the group entry so a group is /// liveness. The monitor's `Down` is delivered to the group reaper's single
/// self-contained: draining the membership tells us whether the member is /// inbox (see [`ProcessGroups::deaths`]), so the membership carries only what
/// still alive, and dropping the membership drops its monitor. /// [`leave`] needs to tear the registration down: the id.
struct Membership { pub(crate) struct Membership {
member: Member, pub(crate) member: Member,
monitor: Monitor, /// `None` for a remote member (cluster): the origin node is its liveness
/// authority; nothing here watches it.
pub(crate) monitor: Option<MonitorId>,
} }
/// The store: `name → multiset<Member>`. Within a single group a `Member` /// The store: `name → multiset<Member>`. Within a single group a `Member`
@@ -202,46 +208,60 @@ struct Membership {
/// ///
/// Locking discipline. Held under one Leaf-class `RawMutex` on `RuntimeInner`, /// Locking discipline. Held under one Leaf-class `RawMutex` on `RuntimeInner`,
/// mirroring the registry, and never held together with another Leaf lock (it /// mirroring the registry, and never held together with another Leaf lock (it
/// never touches the registry or a slot's cold lock). The two operations that /// never touches the registry or a slot's cold lock). Monitor registration and
/// do need another lock are kept off the group-lock path: /// removal take the target's cold lock (also Leaf), so they run *before* /
/// *after* the group lock, never under it — see [`join`] for the ordering that
/// makes that safe. Nothing under this lock ever touches a channel.
/// ///
/// - `monitor()` / `demonitor()` take the target's cold lock (also Leaf), so /// Eviction is *eager*: every membership's monitor delivers to the one
/// they run *before* / *after* the group lock, never under it. /// `deaths` channel, drained by a per-run reaper actor that sweeps the dead
/// - draining a monitor with `try_recv` takes the channel's Channel-class /// pid out of every group the moment its `Down` is scheduled. The read path
/// lock, which the lock order permits *under* a Leaf; a channel critical /// keeps a slot-liveness backstop for the window between a death and the
/// section only does the lock-free unpark protocol, so no Leaf ever nests /// reaper's turn.
/// under it.
///
/// Evicted and rejected [`Monitor`]s are therefore dropped only *after* the
/// group lock is released, so a receiver-drop never runs a wakeup under the
/// lock — the same discipline as `demonitor`.
pub(crate) struct ProcessGroups { pub(crate) struct ProcessGroups {
groups: HashMap<String, Vec<Membership>>, groups: HashMap<String, Vec<Membership>>,
/// The reaper's inboxes: every membership monitor is registered against
/// a clone of `deaths`. `None` until the first `join` of a run spawns
/// the reaper; a stale one (receiver gone with the previous run's
/// teardown) is detected via `receiver_alive` and replaced.
reaper: Option<ReaperInboxes>,
/// `NodeId → node name` for every peer with members in the store, kept
/// by the pg actor under this lock, so a stored remote member can be
/// rendered back to its wire identity without asking anyone.
#[cfg(feature = "cluster")]
node_names: HashMap<NodeId, String>,
} }
impl ProcessGroups { impl ProcessGroups {
pub(crate) fn new() -> Self { pub(crate) fn new() -> Self {
Self { Self {
groups: HashMap::new(), groups: HashMap::new(),
reaper: None,
#[cfg(feature = "cluster")]
node_names: HashMap::new(),
} }
} }
/// Insert `ms` into `group`. Idempotent on the *member*: if the member is /// Forget the reaper. Called at the start of every `run()` so a stopped
/// already present the new membership is handed back (`Some`) so the caller /// reaper from a previous run is never sent to; `join` respawns.
/// can tear its now-redundant monitor down outside the lock; `None` means pub(crate) fn reset_reaper(&mut self) {
/// it was inserted. self.reaper = None;
fn join(&mut self, group: &str, ms: Membership) -> Option<Membership> { }
/// Insert `ms` into `group`. Idempotent on the *member*: `false` means the
/// member was already present and nothing changed; `true` means inserted.
pub(crate) fn join(&mut self, group: &str, ms: Membership) -> bool {
let v = self.groups.entry(group.to_owned()).or_default(); let v = self.groups.entry(group.to_owned()).or_default();
if v.iter().any(|e| e.member == ms.member) { if v.iter().any(|e| e.member == ms.member) {
return Some(ms); return false;
} }
v.push(ms); v.push(ms);
None true
} }
/// Remove `member`'s membership from `group`, returning it (so the caller /// Remove `member`'s membership from `group`, returning it (so the caller
/// can `demonitor` it outside the lock). An emptied group is pruned. /// can unregister its monitor outside the lock). An emptied group is pruned.
fn leave(&mut self, group: &str, member: Member) -> Option<Membership> { pub(crate) fn leave(&mut self, group: &str, member: Member) -> Option<Membership> {
let v = self.groups.get_mut(group)?; let v = self.groups.get_mut(group)?;
let pos = v.iter().position(|e| e.member == member)?; let pos = v.iter().position(|e| e.member == member)?;
let removed = v.remove(pos); let removed = v.remove(pos);
@@ -252,20 +272,23 @@ impl ProcessGroups {
} }
/// The one dumb eviction primitive: drop every member matching `pred` from /// The one dumb eviction primitive: drop every member matching `pred` from
/// every group, pruning emptied groups, and return the evicted memberships' /// every group, pruning emptied groups, and return the evicted
/// monitors for the caller to drop outside the lock. The primitive does not /// memberships with the group each was in. The primitive does not know
/// know *why* a member leaves; that is the caller's concern. Its callers are /// *why* a member leaves; that is the caller's concern. Its callers are
/// the death hook (`reap_group`) and, once clustering lands, an /// the reaper (a local death) and the cluster's node-down / re-sync
/// incarnation-eviction sweep — both over this same predicate path, which is /// sweeps — all over this same predicate path, which is the whole reason
/// the whole reason to shape eviction as a predicate. Insertion order within /// to shape eviction as a predicate. Insertion order within a group is
/// a group is preserved (`members` / `pick` are order-stable). /// preserved (`members` / `pick` are order-stable).
fn remove_where(&mut self, mut pred: impl FnMut(&Member) -> bool) -> Vec<Monitor> { pub(crate) fn remove_where(
&mut self,
mut pred: impl FnMut(&Member) -> bool,
) -> Vec<(String, Membership)> {
let mut evicted = Vec::new(); let mut evicted = Vec::new();
self.groups.retain(|_, v| { self.groups.retain(|g, v| {
let mut i = 0; let mut i = 0;
while i < v.len() { while i < v.len() {
if pred(&v[i].member) { if pred(&v[i].member) {
evicted.push(v.remove(i).monitor); evicted.push((g.clone(), v.remove(i)));
} else { } else {
i += 1; i += 1;
} }
@@ -275,38 +298,6 @@ impl ProcessGroups {
evicted evicted
} }
/// Drain-on-contact death hook. The registry can prune a stale binding
/// lazily, on contact, because it only ever resolves one binding at a time;
/// a group is *iterated* — `members` fans out to everyone — so it must not
/// carry a dead member across a broadcast. Every group operation reaps the
/// group it touches first.
///
/// Drains every membership monitor in `group` with a non-blocking
/// `try_recv`: a delivered `Down` (any reason) or a closed channel means
/// that member is dead. On the first death detected, sweep *all* of the
/// dead pids out of *every* group via [`remove_where`] — a death is removed
/// from each group it joined, not just the one being touched. Returns the
/// evicted monitors to drop outside the lock.
fn reap_group(&mut self, group: &str) -> Vec<Monitor> {
let dead: Vec<Pid> = {
let Some(v) = self.groups.get(group) else {
return Vec::new();
};
v.iter()
.filter_map(|e| match e.monitor.rx.try_recv() {
// A Down arrived, or the channel closed and drained: dead.
Ok(Some(_)) | Err(_) => Some(e.member.pid),
// Empty but open — the sender still lives in the slot: alive.
Ok(None) => None,
})
.collect()
};
if dead.is_empty() {
return Vec::new();
}
self.remove_where(|m| dead.contains(&m.pid))
}
/// Raw enumeration of a group's members — no liveness filtering. Used by /// Raw enumeration of a group's members — no liveness filtering. Used by
/// tests to assert storage state independently of the read-path backstop. /// tests to assert storage state independently of the read-path backstop.
#[cfg(test)] #[cfg(test)]
@@ -317,16 +308,24 @@ impl ProcessGroups {
.unwrap_or_default() .unwrap_or_default()
} }
/// Live members of `group`, in insertion order. The `is_live` oracle is the /// Live members of `group` **on `node`**, in insertion order. The
/// read-path backstop: a member whose slot is already dead is /// `is_live` oracle is the read-path backstop: a member whose slot is
/// dropped from the *result* even if its `Down` has not been drained yet. /// already dead is dropped from the *result* even if the reaper has not
/// Backstop only — the entry stays in storage; eviction is the monitor's /// swept it yet. Backstop only — the entry stays in storage; eviction is
/// job (`reap_group`). /// the reaper's job. The node filter is what keeps the local API local:
fn members_where(&self, group: &str, mut is_live: impl FnMut(Pid) -> bool) -> Vec<Pid> { /// a remote member's `pid` is another node's slot bits, meaningless to
/// `is_live` and to any local send.
fn members_where(
&self,
group: &str,
node: NodeId,
mut is_live: impl FnMut(Pid) -> bool,
) -> Vec<Pid> {
self.groups self.groups
.get(group) .get(group)
.map(|v| { .map(|v| {
v.iter() v.iter()
.filter(|e| e.member.node == node)
.map(|e| e.member.pid) .map(|e| e.member.pid)
.filter(|&p| is_live(p)) .filter(|&p| is_live(p))
.collect() .collect()
@@ -334,19 +333,210 @@ impl ProcessGroups {
.unwrap_or_default() .unwrap_or_default()
} }
/// The first live member of `group` in insertion order — stateless /// The first live member of `group` on `node` in insertion order —
/// first-live `pick`, with the same read-path backstop as `members_where`. /// stateless first-live `pick`, with the same read-path backstop and node
fn first_member_where(&self, group: &str, mut is_live: impl FnMut(Pid) -> bool) -> Option<Pid> { /// filter as `members_where`.
fn first_member_where(
&self,
group: &str,
node: NodeId,
mut is_live: impl FnMut(Pid) -> bool,
) -> Option<Pid> {
self.groups self.groups
.get(group)? .get(group)?
.iter() .iter()
.filter(|e| e.member.node == node)
.map(|e| e.member.pid) .map(|e| e.member.pid)
.find(|&p| is_live(p)) .find(|&p| is_live(p))
} }
} }
/// The store's cluster-side surface: raw reads the pg actor needs to speak
/// for this node (`Sync`, membership checks) and the peer-name memo. One
/// `cfg` block: everything here exists only when there is a mesh.
#[cfg(feature = "cluster")]
impl ProcessGroups {
/// Does `group` hold `member` right now? (Raw storage, no liveness.)
pub(crate) fn contains(&self, group: &str, member: &Member) -> bool {
self.groups
.get(group)
.is_some_and(|v| v.iter().any(|e| e.member == *member))
}
/// Every stored member of `group`, any node, insertion order. Raw storage.
pub(crate) fn all_of(&self, group: &str) -> Vec<Member> {
self.groups
.get(group)
.map(|v| v.iter().map(|e| e.member).collect())
.unwrap_or_default()
}
/// `(group, [pid])` for every group with a member on `node` — the
/// `Sync` payload. Raw storage; groups with no such member are omitted.
pub(crate) fn groups_on(&self, node: NodeId) -> Vec<(String, Vec<Pid>)> {
let mut out: Vec<(String, Vec<Pid>)> = self
.groups
.iter()
.filter_map(|(g, v)| {
let pids: Vec<Pid> = v
.iter()
.filter(|e| e.member.node == node)
.map(|e| e.member.pid)
.collect();
(!pids.is_empty()).then(|| (g.clone(), pids))
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
/// Record / forget the name behind a peer's `NodeId`.
pub(crate) fn set_node_name(&mut self, node: NodeId, name: String) {
self.node_names.insert(node, name);
}
pub(crate) fn forget_node_name(&mut self, node: NodeId) {
self.node_names.remove(&node);
}
pub(crate) fn node_name(&self, node: NodeId) -> Option<&str> {
self.node_names.get(&node).map(String::as_str)
}
}
/// The group reaper: one detached actor per run, spawned by the first `join`,
/// parked on the shared `deaths` inbox. Every local membership's monitor
/// delivers here, so a death is swept out of *every* group it joined as soon
/// as the reaper is scheduled — no group operation has to happen first.
/// Sweeps by `(node, pid)`: only local members, since a remote member's pid
/// bits are meaningless here. Exits when the last sender is gone, i.e. never
/// during a run (the store holds one); the run's teardown stops it like any
/// other parked actor. Spawned under `ROOT_PID` so its exit signal is absorbed
/// rather than delivered to whichever supervisor's child happened to join
/// first.
///
/// Under `cluster` the same actor is the node's **pg actor** (RFC 010 Phase
/// 5, c15): it also drains a control inbox of local join/leave announcements,
/// the membership stream and the exposed `"pg"` inbox — see
/// [`crate::cluster::pg`]. Its store-side sweep is unchanged.
#[cfg(not(feature = "cluster"))]
fn reaper(rx: crate::channel::Receiver<Down>, ctl: crate::channel::Receiver<PgEvent>) {
// No mesh: nothing to tell about joins/leaves. Drop the control inbox
// so announcements are refused at the sender rather than queued.
drop(ctl);
while let Ok(down) = rx.recv() {
sweep_local_death(down.pid);
// Evicted memberships hold only ids; their monitors have fired.
}
}
/// What the local API tells the reaper besides deaths (which arrive as
/// [`Down`] on their own inbox — that channel's type is fixed by the monitor
/// primitive, so the two cannot be one enum). The default reaper has no use
/// for these; the cluster's pg actor broadcasts them (RFC 010 Phase 5).
// The default reaper never looks inside — that is the point, not a bug.
#[cfg_attr(not(feature = "cluster"), allow(dead_code))]
pub(crate) enum PgEvent {
/// `join` inserted `pid` into `group`. The consumer re-checks the store
/// before acting on it.
Joined { group: String, pid: Pid },
/// `leave` removed `pid` from `group`.
Left { group: String, pid: Pid },
/// `cluster::start` has the manager up and the local identity set: take
/// a membership subscription, register + expose the `"pg"` name, and
/// start speaking to peers.
#[cfg(feature = "cluster")]
Attach,
}
/// Evict the local member `pid` from every group. The reaper's one store
/// operation; returns what was evicted with its group (the cluster's
/// `Leave` broadcast wants both).
pub(crate) fn sweep_local_death(pid: Pid) -> Vec<(String, Membership)> {
with_runtime(|inner| {
let node = inner.node_id;
inner
.process_groups
.lock()
.remove_where(|m| m.node == node && m.pid == pid)
})
}
/// The reaper's inboxes. `deaths` is the liveness authority for the set
/// (`ctl` is created and dropped with it, on the same actor).
#[derive(Clone)]
pub(crate) struct ReaperInboxes {
pub(crate) deaths: Sender<Down>,
/// The control inbox: local `join`/`leave` announce here (see
/// [`PgEvent`]). The default reaper closes it on entry.
pub(crate) ctl: Sender<PgEvent>,
}
impl ReaperInboxes {
fn alive(&self) -> bool {
self.deaths.receiver_alive()
}
}
/// Live senders for the reaper's inboxes, spawning the reaper if this run has
/// none yet. Two racing first-spawns may both spawn; the loser's senders drop
/// on return, its spare reaper sees a closed inbox and exits.
pub(crate) fn reaper_inboxes() -> ReaperInboxes {
let existing = with_runtime(|inner| {
let pg = inner.process_groups.lock();
pg.reaper.clone().filter(ReaperInboxes::alive)
});
if let Some(r) = existing {
return r;
}
let (tx, rx) = channel::<Down>();
let (ctl_tx, ctl_rx) = channel::<PgEvent>();
// Detached: the handle drops here. The reaper's lifetime is the run's.
// The ONE seam between the local store and the cluster: same inboxes,
// different body.
#[cfg(not(feature = "cluster"))]
let _ = spawn_under(crate::runtime::ROOT_PID, move || reaper(rx, ctl_rx));
#[cfg(feature = "cluster")]
let _ = spawn_under(crate::runtime::ROOT_PID, move || {
crate::cluster::pg::actor(rx, ctl_rx)
});
let fresh = ReaperInboxes {
deaths: tx,
ctl: ctl_tx,
};
with_runtime(|inner| {
let mut pg = inner.process_groups.lock();
match &pg.reaper {
Some(r) if r.alive() => r.clone(),
_ => {
pg.reaper = Some(fresh.clone());
fresh
}
}
})
}
/// A live sender for the reaper's `deaths` inbox (spawning it if needed).
fn deaths_sender() -> Sender<Down> {
reaper_inboxes().deaths
}
/// Announce a local group change to the reaper, if this run has one. A
/// closed inbox is the default reaper (uninterested) or a run tearing down.
fn announce(msg: PgEvent) {
let ctl = with_runtime(|inner| {
inner
.process_groups
.lock()
.reaper
.as_ref()
.map(|r| r.ctl.clone())
});
if let Some(ctl) = ctl {
let _ = ctl.send(msg);
}
}
/// Build the full member identity for `pid` from runtime identity. /// Build the full member identity for `pid` from runtime identity.
fn member_for(inner: &crate::runtime::RuntimeInner, pid: Pid) -> Member { pub(crate) fn member_for(inner: &crate::runtime::RuntimeInner, pid: Pid) -> Member {
Member { Member {
node: inner.node_id, node: inner.node_id,
incarnation: inner.incarnation, incarnation: inner.incarnation,
@@ -358,7 +548,7 @@ fn member_for(inner: &crate::runtime::RuntimeInner, pid: Pid) -> Member {
/// no lock — identical to the registry's guard. The read-path backstop: a /// no lock — identical to the registry's guard. The read-path backstop: a
/// generation is never reused, so a dead member is detectable independently of /// generation is never reused, so a dead member is detectable independently of
/// whether its monitor `Down` has been drained yet. /// whether its monitor `Down` has been drained yet.
fn live(inner: &crate::runtime::RuntimeInner, pid: Pid) -> bool { pub(crate) fn live(inner: &crate::runtime::RuntimeInner, pid: Pid) -> bool {
inner.slot_at(pid).is_some_and(|s| s.is_live_for(pid)) inner.slot_at(pid).is_some_and(|s| s.is_live_for(pid))
} }
@@ -367,61 +557,76 @@ fn live(inner: &crate::runtime::RuntimeInner, pid: Pid) -> bool {
/// added the membership, `false` if it was already a member. /// added the membership, `false` if it was already a member.
/// ///
/// Installs a monitor on `pid` so the actor's death evicts it from the group /// Installs a monitor on `pid` so the actor's death evicts it from the group
/// automatically — you never have to remove a dead member yourself. A redundant /// automatically — you never have to remove a dead member yourself. Joining a
/// (idempotent) join tears its extra monitor back down. /// pid that is already dead is accepted and evicted the same way (via a
/// `NoProc` notice), so it never shows up in a read.
/// ///
/// Panics if called outside `Runtime::run()`. /// Panics if called outside `Runtime::run()`.
pub fn join<A>(group: impl Into<String>, pid: Pid<A>) -> bool { pub fn join<A>(group: impl Into<String>, pid: Pid<A>) -> bool {
let group = group.into(); let group = group.into();
let pid = pid.erase(); let pid = pid.erase();
// Install the monitor BEFORE taking the group lock: monitor() acquires the let deaths = deaths_sender();
// target's cold lock (Leaf), and two Leaf locks are never held at once. The // Record the membership BEFORE arming its monitor: the reaper sweeps by
// registration races `finalize_actor` under that cold lock exactly as every // pid on the first `Down`, so a `Down` that could precede the entry would
// other monitor does, so no death can slip between the join and the monitor // leave a corpse in storage forever (visible to no read — the backstop
// being in place. // hides it — but a leak, and once groups are clustered a member that
let mon = monitor(pid); // would be announced). Arming after insertion means every `Down` finds
// its entry. The monitor id is allocated up front so `leave` can tear the
let (rejected, reaped) = with_runtime(|inner| { // registration down even if it lands in the tiny window before arming (an
// orphaned registration is harmless: its `Down` names a pid whose
// membership is gone, and the sweep finds nothing).
let id = with_runtime(|inner| inner.alloc_monitor_id());
let inserted = with_runtime(|inner| {
let ms = Membership { let ms = Membership {
member: member_for(inner, pid), member: member_for(inner, pid),
monitor: mon, monitor: Some(id),
}; };
let mut pg = inner.process_groups.lock(); inner.process_groups.lock().join(&group, ms)
let reaped = pg.reap_group(&group); });
let rejected = pg.join(&group, ms); if !inserted {
(rejected, reaped) return false;
}
// Tell the reaper (the cluster's pg actor re-checks the store before it
// broadcasts, so a `leave`/death that overtakes this announcement is
// never advertised as a join).
announce(PgEvent::Joined {
group: group.clone(),
pid,
}); });
// Outside the group lock: drop the reaped (dead) monitors, and if this join // Outside the group lock: registration takes the target's cold lock (Leaf).
// was redundant, demonitor + drop the extra monitor we just installed. // The registration races `finalize_actor` under that cold lock exactly as
drop(reaped); // every other monitor does, so no death can slip between the join and the
match rejected { // monitor being in place.
Some(dup) => { if !register_monitor(pid, id, &deaths) {
demonitor(&dup.monitor); // Already gone: queue the notice ourselves, exactly as `monitor` does.
false let _ = deaths.send(Down {
} pid,
None => true, reason: DownReason::NoProc,
});
} }
true
} }
/// Drop `pid`'s membership of `group`. Returns whether a membership was /// Drop `pid`'s membership of `group`. Returns whether a membership was
/// removed. The membership's monitor is demonitored and dropped. /// removed. The membership's monitor registration is torn down.
/// ///
/// Panics if called outside `Runtime::run()`. /// Panics if called outside `Runtime::run()`.
pub fn leave<A>(group: &str, pid: Pid<A>) -> bool { pub fn leave<A>(group: &str, pid: Pid<A>) -> bool {
let pid = pid.erase(); let pid = pid.erase();
let (removed, reaped) = with_runtime(|inner| { let removed = with_runtime(|inner| {
let member = member_for(inner, pid); let member = member_for(inner, pid);
let mut pg = inner.process_groups.lock(); inner.process_groups.lock().leave(group, member)
let reaped = pg.reap_group(group);
let removed = pg.leave(group, member);
(removed, reaped)
}); });
drop(reaped);
match removed { match removed {
Some(ms) => { Some(ms) => {
demonitor(&ms.monitor); if let Some(id) = ms.monitor {
unregister_monitor(pid, id);
}
announce(PgEvent::Left {
group: group.to_owned(),
pid,
});
true true
} }
None => false, None => false,
@@ -431,21 +636,19 @@ pub fn leave<A>(group: &str, pid: Pid<A>) -> bool {
/// Every live member of `group`, in the order they joined. Returns an empty /// Every live member of `group`, in the order they joined. Returns an empty
/// vector if the group does not exist or has no live members. /// vector if the group does not exist or has no live members.
/// ///
/// Dead members are never returned: the group is pruned of anything that has /// Dead members are never returned: the reaper evicts a member as soon as its
/// died before the read, and as a backstop a member whose slot is already dead /// death is processed, and as a backstop a member whose slot is already dead
/// is dropped from the result even in the brief window before its death has /// is dropped from the result even in the brief window before the reaper's
/// been fully processed. /// turn.
/// ///
/// Panics if called outside `Runtime::run()`. /// Panics if called outside `Runtime::run()`.
pub fn members(group: &str) -> Vec<Pid> { pub fn members(group: &str) -> Vec<Pid> {
let (pids, reaped) = with_runtime(|inner| { with_runtime(|inner| {
let mut pg = inner.process_groups.lock(); inner
let reaped = pg.reap_group(group); .process_groups
let pids = pg.members_where(group, |pid| live(inner, pid)); .lock()
(pids, reaped) .members_where(group, inner.node_id, |pid| live(inner, pid))
}); })
drop(reaped);
pids
} }
/// One live member of `group`, or `None` if the group is empty (or every /// One live member of `group`, or `None` if the group is empty (or every
@@ -455,14 +658,12 @@ pub fn members(group: &str) -> Vec<Pid> {
/// ///
/// Panics if called outside `Runtime::run()`. /// Panics if called outside `Runtime::run()`.
pub fn pick(group: &str) -> Option<Pid> { pub fn pick(group: &str) -> Option<Pid> {
let (picked, reaped) = with_runtime(|inner| { with_runtime(|inner| {
let mut pg = inner.process_groups.lock(); inner
let reaped = pg.reap_group(group); .process_groups
let picked = pg.first_member_where(group, |pid| live(inner, pid)); .lock()
(picked, reaped) .first_member_where(group, inner.node_id, |pid| live(inner, pid))
}); })
drop(reaped);
picked
} }
/// Typed [`pick`]: one live member of `group` as a [`Pid<A>`](Pid). /// Typed [`pick`]: one live member of `group` as a [`Pid<A>`](Pid).
@@ -505,8 +706,8 @@ pub fn dispatch<A: Addressable>(group: &str, msg: A::Msg) -> Result<Pid<A>, Send
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::channel::{channel, Sender}; use crate::scheduler::spawn;
use crate::monitor::{Down, DownReason, MonitorId}; use std::time::{Duration, Instant};
fn member(index: u32, generation: u32) -> Member { fn member(index: u32, generation: u32) -> Member {
Member { Member {
@@ -516,33 +717,21 @@ mod tests {
} }
} }
/// A synthetic membership with a real (but slot-less) monitor channel. The /// A synthetic membership: the store never looks at the id.
/// returned `Sender` stands in for the slot's `Down` sender: hold it to fn synth(index: u32, generation: u32) -> Membership {
/// keep the member "alive" (`try_recv` → `Ok(None)`), `send` a `Down` to Membership {
/// simulate death, or `drop` it to simulate a drained/closed channel.
fn synth(index: u32, generation: u32) -> (Membership, Sender<Down>) {
let pid = Pid::new(index, generation);
let (tx, rx) = channel::<Down>();
let ms = Membership {
member: member(index, generation), member: member(index, generation),
monitor: Monitor { monitor: Some(MonitorId(0)),
id: MonitorId(0), }
target: pid,
rx,
},
};
(ms, tx)
} }
#[test] #[test]
fn join_is_idempotent_within_a_group() { fn join_is_idempotent_within_a_group() {
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let (a, _ta) = synth(1, 0); assert!(pg.join("workers", synth(1, 0)), "first join inserts");
let (b, _tb) = synth(1, 0);
assert!(pg.join("workers", a).is_none(), "first join inserts");
assert!( assert!(
pg.join("workers", b).is_some(), !pg.join("workers", synth(1, 0)),
"second identical join is handed back" "second identical join is refused"
); );
assert_eq!(pg.members_of("workers"), vec![member(1, 0)]); assert_eq!(pg.members_of("workers"), vec![member(1, 0)]);
} }
@@ -550,12 +739,9 @@ mod tests {
#[test] #[test]
fn same_pid_in_many_groups_is_independent() { fn same_pid_in_many_groups_is_independent() {
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let (a, _ta) = synth(1, 0); pg.join("a", synth(1, 0));
let (b, _tb) = synth(1, 0); pg.join("b", synth(1, 0));
let (c, _tc) = synth(2, 0); pg.join("b", synth(2, 0));
pg.join("a", a);
pg.join("b", b);
pg.join("b", c);
assert_eq!(pg.members_of("a"), vec![member(1, 0)]); assert_eq!(pg.members_of("a"), vec![member(1, 0)]);
assert_eq!(pg.members_of("b"), vec![member(1, 0), member(2, 0)]); assert_eq!(pg.members_of("b"), vec![member(1, 0), member(2, 0)]);
} }
@@ -564,11 +750,9 @@ mod tests {
fn distinct_generations_are_distinct_members() { fn distinct_generations_are_distinct_members() {
// ABA guard: same slot index, different generation = different actor. // ABA guard: same slot index, different generation = different actor.
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let (a, _ta) = synth(1, 0); assert!(pg.join("g", synth(1, 0)));
let (b, _tb) = synth(1, 1);
assert!(pg.join("g", a).is_none());
assert!( assert!(
pg.join("g", b).is_none(), pg.join("g", synth(1, 1)),
"different generation is a distinct member" "different generation is a distinct member"
); );
assert_eq!(pg.members_of("g"), vec![member(1, 0), member(1, 1)]); assert_eq!(pg.members_of("g"), vec![member(1, 0), member(1, 1)]);
@@ -577,10 +761,8 @@ mod tests {
#[test] #[test]
fn leave_removes_one_membership_and_prunes_empty_groups() { fn leave_removes_one_membership_and_prunes_empty_groups() {
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let (a, _ta) = synth(1, 0); pg.join("g", synth(1, 0));
let (b, _tb) = synth(2, 0); pg.join("g", synth(2, 0));
pg.join("g", a);
pg.join("g", b);
assert!(pg.leave("g", member(1, 0)).is_some()); assert!(pg.leave("g", member(1, 0)).is_some());
assert_eq!(pg.members_of("g"), vec![member(2, 0)]); assert_eq!(pg.members_of("g"), vec![member(2, 0)]);
assert!( assert!(
@@ -598,7 +780,7 @@ mod tests {
#[test] #[test]
fn remove_where_sweeps_every_group() { fn remove_where_sweeps_every_group() {
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
for (g, (m, _t)) in [ for (g, m) in [
("a", synth(1, 0)), ("a", synth(1, 0)),
("a", synth(2, 0)), ("a", synth(2, 0)),
("b", synth(1, 0)), ("b", synth(1, 0)),
@@ -616,104 +798,137 @@ mod tests {
#[test] #[test]
fn remove_where_can_match_an_incarnation_sweep() { fn remove_where_can_match_an_incarnation_sweep() {
// Shape check for the later evict_incarnation(node, inc) caller. // Shape check for the node-down / incarnation sweep caller.
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let pid = Pid::new(1, 0); let stale = Membership {
let (tx, rx) = channel::<Down>();
let dead = Membership {
member: Member { member: Member {
node: DEFAULT_NODE_ID, node: DEFAULT_NODE_ID,
incarnation: Incarnation::new(7), incarnation: Incarnation::new(7),
pid, pid: Pid::new(1, 0),
},
monitor: Monitor {
id: MonitorId(0),
target: pid,
rx,
}, },
monitor: Some(MonitorId(0)),
}; };
let _keep = tx; pg.join("g", stale);
let (live, _tl) = synth(2, 0); pg.join("g", synth(2, 0));
pg.join("g", dead);
pg.join("g", live);
let evicted = pg.remove_where(|mem| mem.incarnation == Incarnation::new(7)); let evicted = pg.remove_where(|mem| mem.incarnation == Incarnation::new(7));
assert_eq!(evicted.len(), 1); assert_eq!(evicted.len(), 1);
assert_eq!(pg.members_of("g"), vec![member(2, 0)]); assert_eq!(pg.members_of("g"), vec![member(2, 0)]);
} }
#[test] #[test]
fn reap_keeps_live_members() { fn read_backstop_hides_a_member_the_reaper_has_not_yet_swept() {
let mut pg = ProcessGroups::new(); let mut pg = ProcessGroups::new();
let (a, _ta) = synth(1, 0); // sender held: member stays alive pg.join("g", synth(1, 0));
pg.join("a", a); pg.join("g", synth(2, 0));
assert!(pg.reap_group("a").is_empty(), "no deaths");
assert_eq!(pg.members_of("a"), vec![member(1, 0)]);
}
#[test]
fn reap_evicts_a_dead_member_and_sweeps_all_its_groups() {
let mut pg = ProcessGroups::new();
let (a1, ta1) = synth(1, 0); // pid 1 in group a
let (a2, _ta2) = synth(2, 0); // pid 2 in group a (stays alive)
let (b1, _tb1) = synth(1, 0); // pid 1 in group b
pg.join("a", a1);
pg.join("a", a2);
pg.join("b", b1);
// pid 1 dies: its group-a monitor receives a Down. Its group-b monitor
// has not — reap must still sweep pid 1 out of b by the pid predicate.
ta1.send(Down {
pid: Pid::new(1, 0),
reason: DownReason::Exit,
})
.unwrap();
let evicted = pg.reap_group("a");
assert_eq!(
evicted.len(),
2,
"pid 1's memberships in both a and b are evicted"
);
assert_eq!(pg.members_of("a"), vec![member(2, 0)]);
assert!(pg.members_of("b").is_empty(), "swept from b too; pruned");
}
#[test]
fn reap_treats_a_closed_channel_as_dead() {
let mut pg = ProcessGroups::new();
let (a, ta) = synth(1, 0);
pg.join("a", a);
drop(ta); // sender gone, queue empty → try_recv = Err(RecvError) = dead
let evicted = pg.reap_group("a");
assert_eq!(evicted.len(), 1);
assert!(pg.members_of("a").is_empty());
}
#[test]
fn read_backstop_hides_a_member_the_monitor_has_not_yet_reaped() {
let mut pg = ProcessGroups::new();
// Both senders held: reap_group would see Ok(None) and evict neither.
let (a, _ta) = synth(1, 0);
let (b, _tb) = synth(2, 0);
pg.join("g", a);
pg.join("g", b);
// The slot-word oracle already reports pid 1 dead (finalize window), // The slot-word oracle already reports pid 1 dead (finalize window),
// ahead of any Down delivery. // ahead of the reaper's turn.
let dead = Pid::new(1, 0); let dead = Pid::new(1, 0);
let oracle = |pid: Pid| pid != dead; let oracle = |pid: Pid| pid != dead;
assert_eq!( assert_eq!(
pg.members_where("g", oracle), pg.members_where("g", DEFAULT_NODE_ID, oracle),
vec![Pid::new(2, 0)], vec![Pid::new(2, 0)],
"dead pid filtered from read" "dead pid filtered from read"
); );
assert_eq!( assert_eq!(
pg.first_member_where("g", oracle), pg.first_member_where("g", DEFAULT_NODE_ID, oracle),
Some(Pid::new(2, 0)), Some(Pid::new(2, 0)),
"pick skips the dead first member" "pick skips the dead first member"
); );
// Backstop does not evict — that stays the monitor's job; raw storage // Backstop does not evict — that stays the reaper's job; raw storage
// still holds both until reap runs. // still holds both until it runs.
assert_eq!(pg.members_of("g"), vec![member(1, 0), member(2, 0)]); assert_eq!(pg.members_of("g"), vec![member(1, 0), member(2, 0)]);
} }
// ---- reaper: eager eviction against a live runtime ----
/// Raw storage view for a group, bypassing the read-path backstop.
fn stored(group: &str) -> Vec<Member> {
with_runtime(|inner| inner.process_groups.lock().members_of(group))
}
/// Cooperative wait (`smarm::sleep`, never an OS block) until `pred`.
fn wait_until(what: &str, mut pred: impl FnMut() -> bool) {
let deadline = Instant::now() + Duration::from_secs(2);
while !pred() {
assert!(Instant::now() < deadline, "timed out waiting for: {what}");
crate::sleep(Duration::from_millis(1));
}
}
#[test]
fn a_death_is_swept_from_storage_without_any_group_operation() {
crate::run(|| {
let (tx, rx) = channel::<()>();
let w = spawn(move || {
rx.recv().unwrap();
});
let pid = w.pid();
join("a", pid);
join("b", pid);
assert_eq!(stored("a"), vec![member_for_test(pid)]);
tx.send(()).unwrap();
w.join().unwrap();
// No members()/pick()/join() on a or b from here on: the reaper
// alone must clear both.
wait_until("reaper sweeps a and b", || {
stored("a").is_empty() && stored("b").is_empty()
});
});
}
#[test]
fn a_dead_at_join_pid_is_swept_from_storage() {
crate::run(|| {
let h = spawn(|| {});
let pid = h.pid();
h.join().unwrap();
assert!(join("late", pid), "join is accepted; eviction is uniform");
wait_until("reaper sweeps the NoProc member", || {
stored("late").is_empty()
});
});
}
#[test]
fn leave_then_death_does_not_disturb_a_rejoined_group() {
// A monitor unregistered by `leave` must not fire later; the pid's
// fresh membership after re-join is swept exactly once, by its own
// monitor, on death.
crate::run(|| {
let (tx, rx) = channel::<()>();
let w = spawn(move || {
rx.recv().unwrap();
});
let pid = w.pid();
join("g", pid);
assert!(leave("g", pid));
assert!(join("g", pid));
assert_eq!(members("g"), vec![pid]);
tx.send(()).unwrap();
w.join().unwrap();
wait_until("reaper sweeps g", || stored("g").is_empty());
});
}
#[test]
fn reaper_is_respawned_for_a_second_run_of_the_same_runtime() {
let rt = crate::runtime::init(crate::runtime::Config::exact(1));
let body = || {
let h = spawn(|| {});
let pid = h.pid();
h.join().unwrap();
join("g", pid);
wait_until("reaper sweeps g", || stored("g").is_empty());
};
rt.run(body);
rt.run(body);
}
fn member_for_test(pid: Pid) -> Member {
with_runtime(|inner| member_for(inner, pid))
}
} }
+41
View File
@@ -292,3 +292,44 @@ mod typed_pid_tests {
assert_send_sync::<Name<CounterMsg>>(); assert_send_sync::<Name<CounterMsg>>();
} }
} }
// ---- RFC 010 c10: pids auto-serialize (cluster feature) ---------------------
/// A local `Pid<A>` serializes as a
/// [`RemotePid<A>`](crate::cluster::remote::RemotePid): the wire form stamps
/// this node's name and incarnation from the ambient runtime, so a pid can
/// sit inside any message field and reply-to needs no ceremony (RFC 010 §3,
/// "sugar not a bear trap"). Serializing a pid also marks it **watchable**
/// — the wire crossing is the cluster's `mark_watchable` set-site (D12), the
/// exact analog of the membrane crossing.
///
/// Must run inside `run()` (the ambient identity lives on the runtime); a
/// runtime without a cluster identity cannot serialize a pid at all — it is
/// a serialize error, surfacing as the send's `Encode` failure — rather than
/// a `("", 0)` stamp that every peer would silently drop.
#[cfg(feature = "cluster")]
impl<A: 'static> serde::Serialize for Pid<A> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
crate::cluster::remote::RemotePid::<A>::from_local(*self)
.ok_or_else(|| serde::ser::Error::custom("pid serialized with no local node identity"))?
.serialize(s)
}
}
/// Deserializing into a `Pid<A>` is the **collapse**: it succeeds only when
/// the wire pid names this very node (name and incarnation both), and is a
/// decode error otherwise — a foreign pid cannot become a local `Pid`.
/// Fields that may hold a pid from anywhere are `RemotePid<A>`.
#[cfg(feature = "cluster")]
impl<'de, A: 'static> serde::Deserialize<'de> for Pid<A> {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let rp = crate::cluster::remote::RemotePid::<A>::deserialize(d)?;
rp.local().ok_or_else(|| {
serde::de::Error::custom(format!(
"pid {}@{} is not local to this node",
rp.index(),
rp.node()
))
})
}
}
+37
View File
@@ -215,6 +215,7 @@ impl<M> std::error::Error for SendError<M> {}
trait ErasedSender: Send { trait ErasedSender: Send {
fn as_any(&self) -> &dyn Any; fn as_any(&self) -> &dyn Any;
fn queued_len(&self) -> usize; fn queued_len(&self) -> usize;
fn receiver_alive(&self) -> bool;
} }
impl<M: Send + 'static> ErasedSender for Sender<M> { impl<M: Send + 'static> ErasedSender for Sender<M> {
@@ -224,6 +225,9 @@ impl<M: Send + 'static> ErasedSender for Sender<M> {
fn queued_len(&self) -> usize { fn queued_len(&self) -> usize {
Sender::queued_len(self) Sender::queued_len(self)
} }
fn receiver_alive(&self) -> bool {
Sender::receiver_alive(self)
}
} }
/// One typed channel of an actor, type-erased. Concretely a `Sender<M>` filed /// One typed channel of an actor, type-erased. Concretely a `Sender<M>` filed
@@ -442,6 +446,18 @@ pub(crate) fn register_with<M: Send + 'static>(
/// name) and [`install`] (which does not). A leftover mailbox at this slot /// name) and [`install`] (which does not). A leftover mailbox at this slot
/// index from a dead prior incarnation (pid mismatch) is replaced wholesale. /// index from a dead prior incarnation (pid mismatch) is replaced wholesale.
/// Caller holds the registry lock and has established that `me` is live. /// Caller holds the registry lock and has established that `me` is live.
///
/// **One channel per message type per actor.** Publishing a second `M`
/// channel on the same live actor replaces the first — and if the first's
/// receiver is still alive, that replacement drops its last sender, closing
/// it, and any `recv`/`select` on it then returns "closed" immediately and
/// forever: a silent hot loop that starves the scheduler. That is never
/// intended, so it panics here (found the hard way in RFC 010 c9, where two
/// `Name<String>`s registered on one actor did exactly this). Replacing a
/// channel whose receiver is already gone is fine (an actor re-registering
/// after dropping its old inbox) and stays silent. To hold two names of the
/// same type, register them from two actors, or bind both names to one
/// cloned sender.
fn publish_channel<M: Send + 'static>(reg: &mut Registry, me: Pid, tx: Sender<M>) { fn publish_channel<M: Send + 'static>(reg: &mut Registry, me: Pid, tx: Sender<M>) {
let mb = reg let mb = reg
.by_index .by_index
@@ -450,6 +466,16 @@ fn publish_channel<M: Send + 'static>(reg: &mut Registry, me: Pid, tx: Sender<M>
if mb.pid != me { if mb.pid != me {
*mb = Mailbox::new(me); *mb = Mailbox::new(me);
} }
if let Some(existing) = mb.channels.get(&TypeId::of::<M>()) {
assert!(
!existing.sender.receiver_alive() || same_channel::<M>(existing, &tx),
"smarm: actor {me:?} already publishes a live channel for message type `{}`; \
a second one would replace and CLOSE the first (its receiver would then \
read as closed forever). Register the second name from another actor, or \
bind both names to a clone of the same sender.",
type_name::<M>()
);
}
mb.channels.insert( mb.channels.insert(
TypeId::of::<M>(), TypeId::of::<M>(),
Channel { Channel {
@@ -459,6 +485,17 @@ fn publish_channel<M: Send + 'static>(reg: &mut Registry, me: Pid, tx: Sender<M>
); );
} }
/// True if `existing` and `tx` are senders of the very same channel (a
/// cloned sender bound under a second name is the sanctioned way to hold two
/// names of one type on one actor).
fn same_channel<M: Send + 'static>(existing: &Channel, tx: &Sender<M>) -> bool {
existing
.sender
.as_any()
.downcast_ref::<Sender<M>>()
.is_some_and(|old| old.same_channel(tx))
}
/// Publish the current actor's `Sender<A::Msg>` into its mailbox **without** /// Publish the current actor's `Sender<A::Msg>` into its mailbox **without**
/// binding a name, and hand back the typed [`Pid<A>`] that addresses this /// binding a name, and hand back the typed [`Pid<A>`] that addresses this
/// actor directly. /// actor directly.
+18
View File
@@ -1041,6 +1041,17 @@ pub(crate) struct RuntimeInner {
/// checks under it read only the atomic slot word, and the eviction path /// checks under it read only the atomic slot word, and the eviction path
/// keeps it off the send path. /// keeps it off the send path.
pub(crate) process_groups: RawMutex<crate::pg::ProcessGroups>, pub(crate) process_groups: RawMutex<crate::pg::ProcessGroups>,
/// RFC 010 c8: the exposure registry (exposed names + type-hash decoders).
/// RawMutex Leaf, same discipline as `process_groups`; decoders run under
/// it and are leaf-only by contract (they decode and send — `send_dyn`
/// takes `registry`, never this). cfg-gated: zero-cost-when-off (c1).
#[cfg(feature = "cluster")]
pub(crate) exposure: RawMutex<crate::cluster::expose::ExposureState>,
/// RFC 010 c9: the outbound table (node name → the connection's
/// dedicated `Sender<Frame>`), manager-maintained. Leaf; the send happens
/// outside the lock. cfg-gated like `exposure`.
#[cfg(feature = "cluster")]
pub(crate) outbound: RawMutex<crate::cluster::remote::Outbound>,
/// Recycled stacks waiting to be reused by the next spawn. /// Recycled stacks waiting to be reused by the next spawn.
pub(crate) stack_pool: RawMutex<Vec<crate::stack::Stack>>, pub(crate) stack_pool: RawMutex<Vec<crate::stack::Stack>>,
/// Maximum number of stacks to retain in the pool. /// Maximum number of stacks to retain in the pool.
@@ -1099,6 +1110,10 @@ impl RuntimeInner {
node_id, node_id,
incarnation, incarnation,
process_groups: RawMutex::new(crate::pg::ProcessGroups::new()), process_groups: RawMutex::new(crate::pg::ProcessGroups::new()),
#[cfg(feature = "cluster")]
exposure: RawMutex::new(crate::cluster::expose::ExposureState::new()),
#[cfg(feature = "cluster")]
outbound: RawMutex::new(crate::cluster::remote::Outbound::new()),
stack_pool: RawMutex::new(Vec::new()), stack_pool: RawMutex::new(Vec::new()),
stack_pool_cap, stack_pool_cap,
stack_reserve: crate::stack::round_to_pages(stack_reserve), stack_reserve: crate::stack::round_to_pages(stack_reserve),
@@ -1423,6 +1438,9 @@ impl Runtime {
// done" — every remaining top-level actor is asked to shut down (see // done" — every remaining top-level actor is asked to shut down (see
// `finalize_actor` / `shutdown_forest_roots`). // `finalize_actor` / `shutdown_forest_roots`).
self.inner.set_root(initial_handle.pid()); self.inner.set_root(initial_handle.pid());
// A previous run's group reaper was stopped with that run; forget it
// so the first `join` of this run spawns a fresh one.
self.inner.process_groups.lock().reset_reaper();
// Launch N-1 extra scheduler threads, named `smarm-sched-{slot}` so // Launch N-1 extra scheduler threads, named `smarm-sched-{slot}` so
// they are identifiable in `/proc/<pid>/task/*/comm`, stack dumps and // they are identifiable in `/proc/<pid>/task/*/comm`, stack dumps and
+9
View File
@@ -80,6 +80,13 @@ mod inner {
// RFC 005 wake slot // RFC 005 wake slot
SlotPush(Pid), // actor-context wake parked in the waking thread's slot SlotPush(Pid), // actor-context wake parked in the waking thread's slot
SlotPop(Pid), // scheduler resumed a pid from its own slot SlotPop(Pid), // scheduler resumed a pid from its own slot
// Cluster (RFC 010): the conn actor's verdict on one inbound frame —
// local knowledge only, never on the wire; the label is
// `InboundVerdict::label()`. No pid: a refused frame has none.
ClusterInbound(&'static str),
// Cluster (RFC 010): the connector's verdict on one dial attempt —
// `"ok"` or `DialError::label()`. No pid.
ClusterDial(&'static str),
} }
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
@@ -291,6 +298,8 @@ mod inner {
Event::Dequeue(p) => ("dequeue".into(), p.index()), Event::Dequeue(p) => ("dequeue".into(), p.index()),
Event::SlotPush(p) => ("slot_push".into(), p.index()), Event::SlotPush(p) => ("slot_push".into(), p.index()),
Event::SlotPop(p) => ("slot_pop".into(), p.index()), Event::SlotPop(p) => ("slot_pop".into(), p.index()),
Event::ClusterInbound(v) => (format!("cluster_inbound {v}"), 0),
Event::ClusterDial(v) => (format!("cluster_dial {v}"), 0),
} }
} }
+117
View File
@@ -0,0 +1,117 @@
//! RFC 010 c6a — connection-actor lifecycle against the manager table.
//!
//! The handshake is bypassed here (c6b wires it): each connection is
//! constructed already-established over a real localhost TCP pair, handed a
//! fabricated `Peer`, and spawned. `spawn_established` registers it with the
//! manager, which takes its handle and monitors it, so the table reflects the
//! connection while it lives and reaps it on any exit path. This proves three
//! things at once: a live connection shows up, a commanded `Disconnect`
//! removes exactly that one, and a peer close (EOF, no command) removes the
//! other.
//!
//! TCP parks the calling actor, so everything runs inside `smarm::run`; the
//! single-threaded runtime is fine because every wait is a cooperative fd park.
#![cfg(feature = "cluster")]
use std::time::Duration;
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::handshake::Peer;
use smarm::cluster::manager::{Call, Manager, Reply, MANAGER};
use smarm::cluster::spawn_established;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{Conn, FramedConn, Transport};
use smarm::cluster::Timing;
use smarm::gen_server::{self, GenServerBuilder};
use smarm::pg::Incarnation;
use smarm::{run, sleep};
/// A fabricated post-handshake peer identity. Only `node_name` matters to the
/// manager table; the rest is filler until c7 consumes it.
fn peer(name: &str) -> Peer {
Peer {
node_name: name.to_string(),
incarnation: Incarnation::new(1),
meta: NodeMeta {
role: "test".to_string(),
region: "test".to_string(),
},
}
}
/// One established transport pair over localhost. Relies on TCP backlog so the
/// sequential dial-then-accept needs no concurrent acceptor (same assumption as
/// the c3 conformance suite).
fn pair(t: &dyn Transport) -> (Box<dyn Conn>, Box<dyn Conn>) {
let mut l = t.listen("127.0.0.1:0").unwrap();
let a = t.dial(&l.local_addr()).unwrap();
let b = l.accept().unwrap();
(a, b)
}
/// Poll the manager until its peer set matches `expected` (sorted), or fail.
/// The bound is generous against a sub-millisecond real cost.
fn wait_peers(expected: &[&str]) {
let want: Vec<String> = expected.iter().map(|s| s.to_string()).collect();
for _ in 0..2000 {
if let Ok(Reply::Peers(got)) = gen_server::call(MANAGER, Call::Peers) {
if got == want {
return;
}
}
sleep(Duration::from_millis(1));
}
let got = gen_server::call(MANAGER, Call::Peers);
panic!("timed out waiting for peers == {want:?}; last = {got:?}");
}
#[test]
fn connection_up_commanded_shutdown_and_eof_all_reflected_in_table() {
run(|| {
// The manager, started plainly and reachable at its well-known name.
// (The supervised subtree in `cluster::start` is permanent by design;
// a plainly-started manager lets this test terminate cleanly.)
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let t = TcpTransport;
let (a1, b1) = pair(&t);
let (a2, b2) = pair(&t);
// Manage the `a` ends as peers node-b and node-c; keep the `b` far ends
// open so neither socket is closed from the far side yet.
spawn_established(FramedConn::new(a1), peer("node-b"), Timing::default())
.expect("node-b registers");
spawn_established(FramedConn::new(a2), peer("node-c"), Timing::default())
.expect("node-c registers");
// Up: both connections register and the table shows them.
wait_peers(&["node-b", "node-c"]);
// A commanded disconnect reaps exactly its own connection: the
// manager drops that entry's handle and the actor stops.
assert!(matches!(
gen_server::call(
MANAGER,
Call::Disconnect {
name: "node-b".to_string()
}
),
Ok(Reply::Disconnected)
));
wait_peers(&["node-c"]);
// A peer close (EOF) reaps the other with no command at all.
drop(b2);
wait_peers(&[]);
// node-b's far end stayed open until here, so its removal above was the
// disconnect command and not an EOF.
drop(b1);
// All connection actors have exited; stop the manager so `run` returns.
mgr.shutdown();
});
}
+180
View File
@@ -0,0 +1,180 @@
//! RFC 010 c6c — heartbeat send + fixed-timeout liveness + teardown.
//!
//! Each case runs one real connection actor over an in-process localhost TCP
//! pair, with the far end held as a raw `FramedConn` (no actor) so the test
//! controls exactly what — if anything — the peer says. That gives the three
//! protocol-visible facts direct handles: heartbeats appear on the wire
//! unprompted; a mute peer is torn down (and reaped from the manager table)
//! once `LIVENESS_TIMEOUT` empties; and a peer that does nothing but send
//! heartbeats keeps the connection alive past that same window.
//!
//! Loopback has no fd and cannot drive liveness (documented on the actor),
//! so everything here is TCP. TCP parks the calling actor, so everything
//! runs inside `smarm::run`.
#![cfg(feature = "cluster")]
use std::time::{Duration, Instant};
use smarm::cluster::conn::{HEARTBEAT_INTERVAL, LIVENESS_TIMEOUT};
use smarm::cluster::envelope::{Frame, NodeMeta};
use smarm::cluster::handshake::Peer;
use smarm::cluster::manager::{Call, Manager, Reply, MANAGER};
use smarm::cluster::spawn_established;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{Conn, FramedConn, Transport};
use smarm::cluster::Timing;
use smarm::gen_server::{self, GenServerBuilder};
use smarm::pg::Incarnation;
use smarm::{run, sleep, spawn};
/// A fabricated post-handshake peer identity (same shape as the c6a suite).
fn peer(name: &str) -> Peer {
Peer {
node_name: name.to_string(),
incarnation: Incarnation::new(1),
meta: NodeMeta {
role: "test".to_string(),
region: "test".to_string(),
},
}
}
/// One established transport pair over localhost (TCP backlog covers the
/// sequential dial-then-accept, as in the c3 conformance suite).
fn pair(t: &dyn Transport) -> (Box<dyn Conn>, Box<dyn Conn>) {
let mut l = t.listen("127.0.0.1:0").unwrap();
let a = t.dial(&l.local_addr()).unwrap();
let b = l.accept().unwrap();
(a, b)
}
fn peers() -> Vec<String> {
match gen_server::call(MANAGER, Call::Peers) {
Ok(Reply::Peers(p)) => p,
other => panic!("manager unreachable: {other:?}"),
}
}
/// Poll until the manager's peer set matches `expected` (sorted) or `budget`
/// runs out.
fn wait_peers(expected: &[&str], budget: Duration) {
let want: Vec<String> = expected.iter().map(|s| s.to_string()).collect();
let deadline = Instant::now() + budget;
while Instant::now() < deadline {
if peers() == want {
return;
}
sleep(Duration::from_millis(10));
}
panic!(
"timed out waiting for peers == {want:?}; last = {:?}",
peers()
);
}
/// The actor emits heartbeats unprompted: the raw far end, saying nothing,
/// sees a `Frame::Heartbeat` well within one interval (the first goes out at
/// spawn).
#[test]
fn heartbeats_are_sent_unprompted() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (a, b) = pair(&TcpTransport);
spawn_established(FramedConn::new(a), peer("hb-send"), Timing::default())
.expect("register");
let mut far = FramedConn::new(b);
let frame = far
.recv_deadline(Instant::now() + HEARTBEAT_INTERVAL)
.expect("a heartbeat before one interval elapses");
assert_eq!(frame, Some(Frame::Heartbeat));
// Teardown: closing the far end is an EOF at the actor.
far.close();
wait_peers(&[], Duration::from_secs(2));
mgr.shutdown();
});
}
/// A mute peer is dead: no inbound frame for `LIVENESS_TIMEOUT` tears the
/// connection down and the manager's monitor reaps the table entry. The
/// entry is still present well inside the window — the teardown is the
/// timer, not an accident of setup.
#[test]
fn mute_peer_is_torn_down_after_liveness_timeout() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (a, b) = pair(&TcpTransport);
spawn_established(FramedConn::new(a), peer("mute"), Timing::default()).expect("register");
// Held open and silent: no frames, no EOF. (Unread inbound
// heartbeats sit in kernel buffers; they are 5 bytes each.)
let _far = FramedConn::new(b);
// Well inside the window the connection is still up.
sleep(LIVENESS_TIMEOUT / 2);
assert_eq!(peers(), vec!["mute".to_string()], "torn down too early");
// ...and once the window empties it is gone. Generous budget over
// the remaining half-window.
wait_peers(&[], LIVENESS_TIMEOUT);
mgr.shutdown();
});
}
/// Heartbeats alone keep a connection alive past `LIVENESS_TIMEOUT`: a far
/// end that sends `Frame::Heartbeat` at the interval (and nothing else)
/// holds the entry; when it goes quiet, liveness finally fires.
#[test]
fn heartbeats_keep_the_connection_alive() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (a, b) = pair(&TcpTransport);
spawn_established(FramedConn::new(a), peer("kept"), Timing::default()).expect("register");
// The far heartbeat pump: interval-paced sends until told to stop,
// then holds the socket open, silent, so the eventual teardown is
// liveness — not EOF.
let (ctl_tx, ctl_rx) = smarm::channel::channel::<()>();
spawn(move || {
let mut far = FramedConn::new(b);
// Phase 1: heartbeat at the interval until the first signal.
while matches!(ctl_rx.try_recv(), Ok(None)) {
far.send(&Frame::Heartbeat).expect("far send");
sleep(HEARTBEAT_INTERVAL);
}
// Phase 2: silent but with the socket held open — dropping
// `far` here would EOF the actor and mask the liveness path.
// Exits when the test's closure ends and drops `ctl_tx` (an
// eternal park would stop `run` from ever returning).
while matches!(ctl_rx.try_recv(), Ok(None)) {
sleep(Duration::from_millis(20));
}
});
// Past the liveness window with margin: still up.
sleep(LIVENESS_TIMEOUT + LIVENESS_TIMEOUT / 2);
assert_eq!(
peers(),
vec!["kept".to_string()],
"liveness fired despite heartbeats"
);
// Silence the pump; liveness now empties and the entry goes.
ctl_tx.send(()).expect("pump alive");
wait_peers(&[], LIVENESS_TIMEOUT * 2);
mgr.shutdown();
// `ctl_tx` drops here, releasing the pump's phase-2 wait.
});
}
+482
View File
@@ -0,0 +1,482 @@
//! RFC 010 c6b — the handshake on the accept/connect path.
//!
//! Path-level tests drive [`dial_handshake`]/[`accept_handshake`] over the
//! loopback transport on plain threads (its intended use — synchronous, no
//! runtime). Integration tests run the manager-backed [`dial`] and
//! [`spawn_acceptor`] over real localhost TCP inside `smarm::run`, and the
//! two-node case as subprocesses via the c4 harness. Flake budget: see
//! tests/common/mod.rs.
#![cfg(feature = "cluster")]
mod common;
use std::sync::mpsc;
use std::time::{Duration, Instant};
use common::{maybe_child, spawn_node, WAIT};
use smarm::cluster::connect::{
accept_handshake, dial, dial_handshake, spawn_acceptor, DialError, HandshakeError,
HANDSHAKE_TIMEOUT,
};
use smarm::cluster::envelope::{Frame, NodeMeta, RejectReason};
use smarm::cluster::handshake::{Local, PeerStanding};
use smarm::cluster::manager::{Call, Manager, Reply, MANAGER};
use smarm::cluster::transport::loopback::LoopbackTransport;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{FramedConn, Transport};
use smarm::cluster::Timing;
use smarm::gen_server::{self, GenServerBuilder};
use smarm::pg::Incarnation;
use smarm::{run, sleep};
const ROLES: &[(&str, fn())] = &[
("hs_listener", role_hs_listener),
("hs_dialer", role_hs_dialer),
];
const HASH: u64 = 0xC6B0_C6B0_C6B0_C6B0;
fn local(name: &str) -> Local {
Local {
node_name: name.into(),
incarnation: Incarnation::new(3),
build_hash: HASH,
meta: NodeMeta {
role: "test".into(),
region: "test".into(),
},
}
}
/// A loopback conn pair as `FramedConn`s, ready for a threaded handshake.
fn loopback_pair() -> (FramedConn, FramedConn) {
let t = LoopbackTransport::default();
let mut l = t.listen("hs").unwrap();
let dialer = FramedConn::new(t.dial("hs").unwrap());
let accepted = FramedConn::new(l.accept().unwrap());
(dialer, accepted)
}
/// Far-future deadline for loopback paths, where it cannot fire anyway.
fn no_deadline() -> Instant {
Instant::now() + Duration::from_secs(3600)
}
/// Park the node forever: it has announced everything the parent asserts on,
/// and must now hold its connection open until SIGKILLed.
fn park() -> ! {
loop {
sleep(Duration::from_secs(1));
}
}
/// Cooperative bounded receive across the closure/actor boundary. A blocking
/// `std::mpsc` wait would park the OS thread and starve the single-threaded
/// scheduler, so every wait inside `run` polls with [`sleep`] instead.
fn poll_recv<T>(rx: &mpsc::Receiver<T>, what: &str) -> T {
let deadline = Instant::now() + WAIT;
loop {
match rx.try_recv() {
Ok(v) => return v,
Err(mpsc::TryRecvError::Empty) => {
assert!(Instant::now() < deadline, "timed out waiting for {what}");
sleep(Duration::from_millis(1));
}
Err(mpsc::TryRecvError::Disconnected) => panic!("channel closed waiting for {what}"),
}
}
}
// ---------------------------------------------------------------------------
// Path level, over loopback on plain threads
// ---------------------------------------------------------------------------
#[test]
fn loopback_happy_path_establishes_both_ends() {
maybe_child(ROLES);
let (mut dialer, mut accepted) = loopback_pair();
let responder = std::thread::spawn(move || {
accept_handshake(
&mut accepted,
local("node-b"),
|name| {
assert_eq!(name, "node-a");
PeerStanding::Free
},
no_deadline(),
)
});
let peer_of_dialer = dial_handshake(&mut dialer, &local("node-a"), no_deadline()).unwrap();
let peer_of_acceptor = responder.join().unwrap().unwrap();
assert_eq!(peer_of_dialer.node_name, "node-b");
assert_eq!(peer_of_acceptor.node_name, "node-a");
}
#[test]
fn loopback_hash_mismatch_rejected_with_frame_then_eof() {
maybe_child(ROLES);
let (mut dialer, mut accepted) = loopback_pair();
let mut wrong = local("node-b");
wrong.build_hash ^= 1;
let responder = std::thread::spawn(move || {
accept_handshake(&mut accepted, wrong, |_| PeerStanding::Free, no_deadline())
});
// The dial side receives the reject frame — the compatibility anchor.
match dial_handshake(&mut dialer, &local("node-a"), no_deadline()) {
Err(HandshakeError::Rejected(RejectReason::HashMismatch)) => {}
other => panic!("expected HashMismatch reject, got {other:?}"),
}
match responder.join().unwrap() {
Err(HandshakeError::Rejected(RejectReason::HashMismatch)) => {}
other => panic!("expected accept side to report the reject, got {other:?}"),
}
}
#[test]
fn loopback_tie_break_loser_closed_silently() {
maybe_child(ROLES);
// The inbound dial is from "node-z"; we are "node-a" with our own dial to
// node-z in flight. dial_wins("node-z", "node-a") is false, so the
// inbound loses: closed with no frame at all.
let (mut dialer, mut accepted) = loopback_pair();
let responder = std::thread::spawn(move || {
accept_handshake(
&mut accepted,
local("node-a"),
|_| PeerStanding::Dialing,
no_deadline(),
)
});
// Silent close: the dial side sees EOF, never a frame.
match dial_handshake(&mut dialer, &local("node-z"), no_deadline()) {
Err(HandshakeError::Closed) => {}
other => panic!("expected silent close (Closed), got {other:?}"),
}
match responder.join().unwrap() {
Err(HandshakeError::TieBreakLoss) => {}
other => panic!("expected TieBreakLoss on the accept side, got {other:?}"),
}
}
#[test]
fn loopback_read_ahead_past_hello_survives_into_established_conn() {
maybe_child(ROLES);
// The buffer trap, proven: the dialer coalesces Hello + Heartbeat before
// the responder's first read, so the Heartbeat lands in the shared
// FramedConn's decode buffer during the handshake. The dialer sends
// nothing afterwards — the post-handshake recv can only succeed if the
// read-ahead travelled with the FramedConn.
let (mut dialer, mut accepted) = loopback_pair();
let (_init, hello) = smarm::cluster::handshake::Initiator::new(&local("node-a"));
dialer.send(&hello).unwrap();
dialer.send(&Frame::Heartbeat).unwrap();
// Both frames are buffered before the responder reads at all.
let (tx, rx) = mpsc::channel();
std::thread::spawn(move || {
let peer = accept_handshake(
&mut accepted,
local("node-b"),
|_| PeerStanding::Free,
no_deadline(),
)
.unwrap();
let next = accepted.recv();
let _ = tx.send((peer, next));
});
// A bounded wait: if the Heartbeat were NOT carried in the buffer, the
// recv above would block forever (the dialer stays open and silent).
let (peer, next) = rx
.recv_timeout(Duration::from_secs(5))
.expect("read-ahead lost: post-handshake recv blocked");
assert_eq!(peer.node_name, "node-a");
match next {
Ok(Some(Frame::Heartbeat)) => {}
other => panic!("expected the read-ahead Heartbeat, got {other:?}"),
}
drop(dialer);
}
// ---------------------------------------------------------------------------
// Deadline + manager integration, over TCP inside the runtime
// ---------------------------------------------------------------------------
#[test]
fn tcp_silent_peer_times_out_on_the_accept_path() {
maybe_child(ROLES);
run(|| {
let t = TcpTransport;
let mut l = t.listen("127.0.0.1:0").unwrap();
// Connect and then say nothing at all.
let silent = t.dial(&l.local_addr()).unwrap();
let mut accepted = FramedConn::new(l.accept().unwrap());
let (tx, rx) = mpsc::channel();
smarm::spawn(move || {
let r = accept_handshake(
&mut accepted,
local("node-b"),
|_| PeerStanding::Free,
Instant::now() + Duration::from_millis(200),
);
let _ = tx.send(r);
});
match poll_recv(&rx, "accept-path outcome") {
Err(HandshakeError::TimedOut) => {}
other => panic!("expected TimedOut, got {other:?}"),
}
drop(silent);
});
}
/// Poll the manager until its peer set matches `expected` (sorted), or fail.
fn wait_peers(expected: &[&str]) {
let want: Vec<String> = expected.iter().map(|s| s.to_string()).collect();
for _ in 0..5000 {
if let Ok(Reply::Peers(got)) = gen_server::call(MANAGER, Call::Peers) {
if got == want {
return;
}
}
sleep(Duration::from_millis(1));
}
let got = gen_server::call(MANAGER, Call::Peers);
panic!("timed out waiting for peers == {want:?}; last = {got:?}");
}
#[test]
fn tcp_duplicate_name_rejected_by_acceptor() {
maybe_child(ROLES);
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let listener = TcpTransport.listen("127.0.0.1:0").unwrap();
let acceptor = spawn_acceptor(listener, local("node-b"), Timing::default());
let addr = acceptor.local_addr().to_string();
// First dial offering "dup-node": establishes and registers.
let mut first = FramedConn::new(TcpTransport.dial(&addr).unwrap());
let peer = dial_handshake(
&mut first,
&local("dup-node"),
Instant::now() + HANDSHAKE_TIMEOUT,
)
.unwrap();
assert_eq!(peer.node_name, "node-b");
wait_peers(&["dup-node"]);
// Second dial offering the same name: deterministic NameTaken.
let mut second = FramedConn::new(TcpTransport.dial(&addr).unwrap());
match dial_handshake(
&mut second,
&local("dup-node"),
Instant::now() + HANDSHAKE_TIMEOUT,
) {
Err(HandshakeError::Rejected(RejectReason::NameTaken)) => {}
other => panic!("expected NameTaken, got {other:?}"),
}
// The established connection was untouched by the rejected one.
wait_peers(&["dup-node"]);
// Teardown: the acceptor owns no connections, so the established one
// is torn down through the table.
acceptor.shutdown();
assert!(matches!(
gen_server::call(
MANAGER,
Call::Disconnect {
name: "dup-node".to_string()
}
),
Ok(Reply::Disconnected)
));
wait_peers(&[]);
first.close();
mgr.shutdown();
});
}
#[test]
fn dial_intent_cleared_when_dialer_dies() {
maybe_child(ROLES);
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (begun_tx, begun_rx) = mpsc::channel();
let (go_tx, go_rx) = mpsc::channel::<()>();
smarm::spawn(move || {
let me = smarm::self_pid();
match gen_server::call(
MANAGER,
Call::DialBegin {
name: "ghost".into(),
pid: me,
},
) {
Ok(Reply::DialBegan(true)) => {}
other => panic!("DialBegin failed: {other:?}"),
}
let _ = begun_tx.send(());
let () = poll_recv(&go_rx, "go signal");
panic!("dialer dies mid-dial");
});
poll_recv(&begun_rx, "DialBegin done");
// While the dialer lives, the intent is visible.
match gen_server::call(
MANAGER,
Call::Standing {
peer_name: "ghost".into(),
},
) {
Ok(Reply::Standing(s)) => assert_eq!(s, PeerStanding::Dialing),
other => panic!("PeerStanding failed: {other:?}"),
}
// Kill it; the monitor must clear the intent without cooperation.
go_tx.send(()).unwrap();
let deadline = Instant::now() + WAIT;
loop {
match gen_server::call(
MANAGER,
Call::Standing {
peer_name: "ghost".into(),
},
) {
Ok(Reply::Standing(s)) if s != PeerStanding::Dialing => break,
_ if Instant::now() > deadline => {
panic!("dial intent not cleared after dialer death")
}
_ => sleep(Duration::from_millis(1)),
}
}
mgr.shutdown();
});
}
// ---------------------------------------------------------------------------
// Two nodes, two processes: the integrated dial against a real acceptor
// ---------------------------------------------------------------------------
/// Announce, then park forever. Neither role ever tears its connection
/// down: a table entry only exists while the *peer* holds its side open, so
/// any teardown here would retract the other node's observation before it
/// had made it. The parent reaps both with SIGKILL once it has both
/// announcements (see [`common::Node`]'s `Drop`).
fn role_hs_listener() {
run(|| {
let _mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let listener = TcpTransport.listen("127.0.0.1:0").unwrap();
let acceptor = spawn_acceptor(listener, local("node-b"), Timing::default());
println!("LISTENING {}", acceptor.local_addr());
wait_peers(&["node-a"]);
println!("PEERS node-a");
park();
});
}
fn role_hs_dialer() {
let addr = std::env::var("SMARM_PEER_ADDR").expect("SMARM_PEER_ADDR not set");
run(move || {
let _mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (tx, rx) = mpsc::channel();
smarm::spawn(move || {
let r = dial(
&TcpTransport,
&addr,
"node-b",
&local("node-a"),
Timing::default(),
);
let _ = tx.send(r);
});
if let Err(e) = poll_recv(&rx, "dial outcome") {
println!("DIAL failed: {e:?}");
std::process::exit(3);
}
wait_peers(&["node-b"]);
println!("PEERS node-b");
park();
});
}
#[test]
fn two_node_integrated_handshake_over_tcp() {
maybe_child(ROLES);
let mut listener = spawn_node("hs_listener", &[]);
let addr = listener.wait_listening();
let mut dialer = spawn_node("hs_dialer", &[("SMARM_PEER_ADDR", &addr)]);
// Each node reports its own table naming the other: a real dial against a
// real acceptor established in both directions. Both nodes then park —
// clean-exit behaviour is the c4 harness's own smoke test, and demanding
// it here would mean a teardown, which is exactly what cannot be ordered
// safely across two processes. Dropping the nodes SIGKILLs them.
dialer.wait_line("PEERS node-b", |l| l == "PEERS node-b");
listener.wait_line("PEERS node-a", |l| l == "PEERS node-a");
}
// ---------------------------------------------------------------------------
// Integrated-dial guardrails (no acceptor involved)
// ---------------------------------------------------------------------------
#[test]
fn concurrent_dial_to_same_name_refused() {
maybe_child(ROLES);
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let (begun_tx, begun_rx) = mpsc::channel();
let (go_tx, go_rx) = mpsc::channel::<()>();
// First dialer parks with the intent held (it never connects —
// 'holding the intent' is all this test needs from it).
smarm::spawn(move || {
let me = smarm::self_pid();
assert!(matches!(
gen_server::call(
MANAGER,
Call::DialBegin {
name: "node-x".into(),
pid: me,
}
),
Ok(Reply::DialBegan(true))
));
let _ = begun_tx.send(());
let () = poll_recv(&go_rx, "go signal");
let _ = gen_server::call(
MANAGER,
Call::DialEnd {
name: "node-x".into(),
},
);
});
poll_recv(&begun_rx, "DialBegin done");
// Second integrated dial to the same name: refused before connecting
// (the addr is unroutable on purpose — it must never be dialed).
let (tx, rx) = mpsc::channel();
smarm::spawn(move || {
let r = dial(
&TcpTransport,
"127.0.0.1:1",
"node-x",
&local("node-a"),
Timing::default(),
);
let _ = tx.send(r);
});
match poll_recv(&rx, "second dial outcome") {
Err(DialError::AlreadyDialing) => {}
other => panic!("expected AlreadyDialing, got {other:?}"),
}
go_tx.send(()).unwrap();
mgr.shutdown();
});
}
+115
View File
@@ -0,0 +1,115 @@
//! RFC 010 — a seed whose address answers as a *different* name
//! (`DialError::PeerNameMismatch`) is dialed once and then parked: the
//! connector must not redial it on backoff forever.
//!
//! Observed from the misdialed peer: each such dial establishes at the
//! responder (it registers, `node_up`), then the dialer closes on the name
//! check (`node_down`) — one membership blip per attempt. Cross-process: a
//! *server* named `server` subscribes and reports; a *client* on fast
//! timing (50–500ms backoff) seeds `("wrongname", server_addr)`. After the
//! first blip the server counts further `NodeUp`s across 2s — several
//! backoff periods. Parked ⇒ zero. Negative-control-verified: with the park
//! stubbed out the count is ≥ 1 in the same window.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::{start, Config, StaticSeeds, Timing};
use std::time::{Duration, Instant};
const ROLES: &[(&str, fn())] = &[("server", role_server), ("client", role_client)];
fn meta() -> NodeMeta {
NodeMeta {
role: "mismatch".into(),
region: "local".into(),
}
}
fn timing() -> Timing {
Timing {
initial_backoff: Duration::from_millis(50),
max_backoff: Duration::from_millis(500),
..Timing::default()
}
}
fn role_server() {
smarm::run(|| {
let cluster = start(Config {
node_name: "server".into(),
meta: meta(),
listen_addr: std::env::var("SMARM_LISTEN_ADDR")
.unwrap_or_else(|_| "127.0.0.1:0".into()),
strategy: Box::new(StaticSeeds::new(Vec::<(String, String)>::new())),
timing: timing(),
})
.expect("binds");
let ev = subscribe().unwrap();
println!("LISTENING {}", cluster.local_addr());
// First blip: the misdialed client establishes, then closes on us.
loop {
match ev.rx.recv() {
Ok(NodeEvent::NodeDown(i)) if i.name == "client" => break,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
println!("BLIP");
// Now count further NodeUps across several backoff periods.
let mut more = 0usize;
let t0 = Instant::now();
while t0.elapsed() < Duration::from_millis(2000) {
match ev.rx.try_recv() {
Ok(Some(NodeEvent::NodeUp(i))) if i.name == "client" => more += 1,
Ok(_) => {}
Err(_) => panic!("manager gone"),
}
smarm::sleep(Duration::from_millis(50));
}
println!("MORE {more}");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
fn role_client() {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
smarm::run(move || {
let _cluster = start(Config {
node_name: "client".into(),
meta: meta(),
listen_addr: "127.0.0.1:0".into(),
strategy: Box::new(StaticSeeds::new(vec![(
"wrongname".to_string(),
server_addr,
)])),
timing: timing(),
})
.expect("binds");
println!("CLIENT UP");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
#[test]
fn mismatched_seed_is_dialed_once_then_parked() {
maybe_child(ROLES);
let mut server = spawn_node("server", &[]);
let saddr = server.wait_listening();
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
client.wait_line("CLIENT UP", |l| l == "CLIENT UP");
server.wait_line("BLIP", |l| l == "BLIP");
let line = server.wait_line("MORE", |l| l.starts_with("MORE "));
let more: usize = line.split_whitespace().nth(1).unwrap().parse().unwrap();
assert_eq!(
more, 0,
"mismatched seed was redialed {more}× after being parked"
);
}
+379
View File
@@ -0,0 +1,379 @@
//! RFC 010 c13 — connection-loss synthesis.
//!
//! Local suite (`run()`, no network): the read-side backstop. A
//! `RemoteMonitor` whose channel closes without a notice reads as
//! `Disconnected` exactly once (a `Monitor` command that reached the conn
//! actor's inbox but was never processed — the drain gap); after
//! `demonitor_remote` a closed channel stays a plain `Err`, never a notice.
//!
//! Cross-process: the headline contrast — an actor's own death gives its
//! TRUE reason, loss of the LINK gives `Disconnected` (both a commanded
//! `Disconnect` and a SIGKILLed peer process are `Disconnected` from the
//! monitor's view: nobody is left to say otherwise). Reconnect does not
//! resurrect: the old monitor yields nothing more, proven by stream ORDER
//! (a fresh monitor over the new link delivers first). The ignored test
//! trips liveness by SIGSTOP and then drops the link too, asserting exactly
//! one notice for one monitor.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::expose::{expose, expose_type};
use smarm::cluster::manager::{Call, Reply, MANAGER};
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::remote::{
self, demonitor_remote, monitor_remote, send_to_remote, RemoteName, RemotePid,
};
use smarm::cluster::{start, Config, RemoteDownReason, StaticSeeds, Timing};
use smarm::pg::Incarnation;
use smarm::{
channel, gen_server, install, register, run, spawn, Addressable, DownReason, Erased, Name, Pid,
};
use std::collections::HashMap;
use std::time::Duration;
// ---- message types (hand-rolled serde; the crate is derive-less) ---------
#[derive(Debug)]
struct Ctl {
cmd: String,
reply_to: RemotePid<Client>,
}
#[derive(Debug)]
struct Answer {
text: String,
pid: Option<RemotePid<Erased>>,
}
struct Client;
impl Addressable for Client {
type Msg = Answer;
}
impl serde::Serialize for Ctl {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(2)?;
t.serialize_element(&self.cmd)?;
t.serialize_element(&self.reply_to)?;
t.end()
}
}
impl<'de> serde::Deserialize<'de> for Ctl {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (cmd, reply_to) = <(String, RemotePid<Client>)>::deserialize(d)?;
Ok(Ctl { cmd, reply_to })
}
}
impl serde::Serialize for Answer {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(2)?;
t.serialize_element(&self.text)?;
t.serialize_element(&self.pid)?;
t.end()
}
}
impl<'de> serde::Deserialize<'de> for Answer {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (text, pid) = <(String, Option<RemotePid<Erased>>)>::deserialize(d)?;
Ok(Answer { text, pid })
}
}
// ================= local suite =========================================
/// A `Monitor` command handed to the connection but never processed (its
/// receiver dropped unread) reads as `Disconnected` — once. A second read
/// is the ordinary closed-channel `Err`, so "exactly one notice" holds.
#[test]
fn unread_command_reads_as_disconnected_once() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (probe_tx, _probe_rx) = channel();
let inbox =
remote::bind_outbound_probe_with_monitors("peer", Incarnation::new(5), probe_tx);
let target = RemotePid::<Erased>::from_parts("peer", Incarnation::new(5), 9, 1);
let m = monitor_remote(target.clone());
assert!(
matches!(m.try_recv(), Ok(None)),
"command is in flight, no notice yet"
);
drop(inbox); // the conn actor died with the command unread
let d = m.recv().unwrap();
assert_eq!(d.pid, target);
assert_eq!(d.reason, RemoteDownReason::Disconnected);
assert!(
m.recv().is_err(),
"second read is closed, not a second notice"
);
assert!(m.try_recv().is_err());
});
}
/// After `demonitor_remote`, a closed channel is a closed channel: no
/// notice is synthesized for a monitor the caller cancelled.
#[test]
fn cancelled_monitor_never_synthesizes() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (probe_tx, _probe_rx) = channel();
let inbox =
remote::bind_outbound_probe_with_monitors("peer", Incarnation::new(5), probe_tx);
let target = RemotePid::<Erased>::from_parts("peer", Incarnation::new(5), 9, 1);
let m = monitor_remote(target);
demonitor_remote(&m);
drop(inbox);
assert!(m.recv().is_err());
assert!(m.try_recv().is_err());
});
}
// ================= cross-process ======================================
const ROLES: &[(&str, fn())] = &[
("server", role_server),
("client", role_client),
("client_stop", role_client_stop),
];
const CTL: Name<Ctl> = Name::new("c13.ctl");
fn cfg(name: &str, seeds: Vec<(String, String)>) -> Config {
Config {
node_name: name.to_string(),
meta: NodeMeta {
role: "c13".into(),
region: "local".into(),
},
listen_addr: std::env::var("SMARM_LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1:0".into()),
strategy: Box::new(StaticSeeds::new(seeds)),
timing: timing(),
}
}
/// The p11 knobs make the liveness test fast: both roles of that test are
/// spawned with `SMARM_FAST_TIMING=1` and agree on a 100ms heartbeat /
/// 500ms liveness window. Everything else runs the shipping defaults.
fn timing() -> Timing {
if std::env::var_os("SMARM_FAST_TIMING").is_some() {
Timing {
heartbeat_interval: Duration::from_millis(100),
liveness_timeout: Duration::from_millis(500),
initial_backoff: Duration::from_millis(50),
max_backoff: Duration::from_millis(500),
..Timing::default()
}
} else {
Timing::default()
}
}
fn wait_up(events: &smarm::cluster::membership::MembershipEvents, who: &str) {
loop {
match events.rx.recv() {
Ok(NodeEvent::NodeUp(i)) if i.name == who => return,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
}
fn disconnect(name: &str) {
assert!(matches!(
gen_server::call(
MANAGER,
Call::Disconnect {
name: name.to_string()
}
),
Ok(Reply::Disconnected)
));
}
/// Server: `spawn` ⇒ a parked worker (answer carries its pid);
/// `kill:<index>` releases it, whereupon it returns (Exit).
fn role_server() {
smarm::run(move || {
let cluster = start(cfg("server", vec![])).expect("binds");
println!("LISTENING {}", cluster.local_addr());
let (tx, rx) = channel::<Ctl>();
register(CTL, tx).unwrap();
expose(CTL);
println!("READY");
let mut workers: HashMap<u32, smarm::channel::Sender<()>> = HashMap::new();
loop {
let ctl = rx.recv().unwrap();
println!("CTL {}", ctl.cmd);
let (text, pid): (String, Option<RemotePid<Erased>>) = match ctl.cmd.as_str() {
"spawn" => {
let (go_tx, go_rx) = channel::<()>();
let p: Pid = spawn(move || {
let _ = go_rx.recv();
})
.pid();
workers.insert(p.index(), go_tx);
(
"ok".into(),
Some(RemotePid::from_local(p).expect("identity set")),
)
}
other => {
let idx: u32 = other.strip_prefix("kill:").unwrap().parse().unwrap();
if let Some(go) = workers.remove(&idx) {
let _ = go.send(());
}
("killed".into(), None)
}
};
send_to_remote(ctl.reply_to, Answer { text, pid }).unwrap();
}
});
}
/// Client-side setup shared by both client roles: join, expose the reply
/// path, hand back an `ask` closure and the membership stream.
fn client_setup() -> (
smarm::cluster::Cluster,
smarm::cluster::membership::MembershipEvents,
impl Fn(&str) -> Answer,
) {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
let cluster = start(cfg("client", vec![("server".into(), server_addr)])).expect("binds");
let ev = subscribe().unwrap();
wait_up(&ev, "server");
let (tx, rx) = channel::<Answer>();
let me: Pid<Client> = install::<Client>(tx);
expose_type::<Answer>();
let ask = move |cmd: &str| -> Answer {
remote::send(
RemoteName::new("server", CTL),
Ctl {
cmd: cmd.into(),
reply_to: RemotePid::from_local(me).expect("identity set"),
},
)
.unwrap();
rx.recv().unwrap()
};
(cluster, ev, ask)
}
fn role_client() {
smarm::run(move || {
let (_cluster, ev, ask) = client_setup();
// 1. Headline: actor death ⇒ TRUE reason; link cut ⇒ Disconnected.
let a = ask("spawn").pid.unwrap();
let b = ask("spawn").pid.unwrap();
let ma = monitor_remote(a.clone());
let mb = monitor_remote(b.clone());
ask(&format!("kill:{}", a.index()));
let d = ma.recv().unwrap();
assert_eq!(d.pid, a);
println!("DOWN actor {:?}", d.reason);
disconnect("server");
let d = mb.recv().unwrap();
assert_eq!(d.pid, b);
println!("DOWN link {:?}", d.reason);
// 2. Reconnect does not resurrect. The connector redials on
// node_down; over the NEW link a fresh monitor delivers, while
// the old one (already answered) yields nothing further — order
// proves it, and `b` is even still alive on the server.
wait_up(&ev, "server");
println!("RECONNECTED");
let c = ask("spawn").pid.unwrap();
let mc = monitor_remote(c.clone());
ask(&format!("kill:{}", b.index()));
ask(&format!("kill:{}", c.index()));
assert_eq!(mc.recv().unwrap().reason, DownReason::Exit.into());
let stray = matches!(mb.try_recv(), Ok(Some(_)));
println!("RESURRECT stray={stray}");
// 3. Peer PROCESS killed ⇒ Disconnected too (nobody is left to send
// Down): the parent SIGKILLs the server once it sees the marker.
let e = ask("spawn").pid.unwrap();
let me_ = monitor_remote(e.clone());
println!("KILL SERVER NOW");
let d = me_.recv().unwrap();
assert_eq!(d.pid, e);
println!("DOWN procdeath {:?}", d.reason);
println!("CLIENT DONE");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// The slow role: liveness expiry (peer SIGSTOPped) followed by the link
/// dropping for real (peer SIGKILLed) — one monitor, exactly one notice.
fn role_client_stop() {
smarm::run(move || {
let (_cluster, _ev, ask) = client_setup();
let a = ask("spawn").pid.unwrap();
let ma = monitor_remote(a.clone());
println!("STOP SERVER NOW");
let d = ma.recv().unwrap(); // liveness expiry, ~liveness_timeout
assert_eq!(d.pid, a);
println!("DOWN stopped {:?}", d.reason);
println!("KILL SERVER NOW");
// Give the drop every chance to produce a second notice, then look.
smarm::sleep(Duration::from_secs(1));
let dup = matches!(ma.try_recv(), Ok(Some(_)));
println!("DUPLICATE dup={dup}");
println!("CLIENT DONE");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// The Phase 4 c13 gate: partition vs. death distinguishable; nothing
/// survives reconnect; a dead peer process is a Disconnected too.
#[test]
fn link_loss_is_disconnected_and_does_not_survive_reconnect() {
maybe_child(ROLES);
let mut server = spawn_node("server", &[]);
let saddr = server.wait_listening();
server.wait_line("READY", |l| l == "READY");
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
client.wait_line("DOWN actor Local(Exit)", |l| l == "DOWN actor Local(Exit)");
client.wait_line("DOWN link Disconnected", |l| l == "DOWN link Disconnected");
client.wait_line("RECONNECTED", |l| l == "RECONNECTED");
client.wait_line("RESURRECT stray=false", |l| l == "RESURRECT stray=false");
client.wait_line("KILL SERVER NOW", |l| l == "KILL SERVER NOW");
server.kill();
client.wait_line("DOWN procdeath Disconnected", |l| {
l == "DOWN procdeath Disconnected"
});
client.wait_line("CLIENT DONE", |l| l == "CLIENT DONE");
}
/// Covers the invariant the headline test cannot: liveness expiry and the
/// transport drop both firing for the same connection yield ONE notice.
/// Runs on the fast [`timing`] (both roles) — was `#[ignore]`d at the 4s
/// default until the p11 knobs landed.
#[test]
fn timeout_then_drop_yields_one_notice() {
maybe_child(ROLES);
let fast = ("SMARM_FAST_TIMING", "1");
let mut server = spawn_node("server", &[fast]);
let saddr = server.wait_listening();
server.wait_line("READY", |l| l == "READY");
let mut client = spawn_node("client_stop", &[("SMARM_SERVER_ADDR", &saddr), fast]);
client.wait_line("STOP SERVER NOW", |l| l == "STOP SERVER NOW");
let spid = server.pid().expect("server alive") as libc::pid_t;
assert_eq!(unsafe { libc::kill(spid, libc::SIGSTOP) }, 0);
client.wait_line("DOWN stopped Disconnected", |l| {
l == "DOWN stopped Disconnected"
});
client.wait_line("KILL SERVER NOW", |l| l == "KILL SERVER NOW");
server.kill(); // SIGKILL works on a stopped process; Drop would too
client.wait_line("DUPLICATE dup=false", |l| l == "DUPLICATE dup=false");
client.wait_line("CLIENT DONE", |l| l == "CLIENT DONE");
}
+161
View File
@@ -0,0 +1,161 @@
//! RFC 010 — `Discovery::Withdrawn`: a strategy retracts a candidate and the
//! connector stops dialing it.
//!
//! Cross-process: a plain *server* node, and a *client* whose strategy is a
//! script: announce a decoy `(ghost, addr)` where `addr` is a raw
//! `TcpListener` the client itself holds (an OS thread accepts and
//! immediately closes, so every dial fails at handshake and the connector
//! keeps retrying on backoff — the accept count is the dial count); after a
//! beat, withdraw the decoy and announce the real server. The client waits
//! for the server's `node_up` — which is *after* the withdrawal in the
//! strategy's own stream — then watches the decoy's accept count stay flat
//! across a window longer than the pending backoff. Before withdrawal it
//! must have been climbing (≥ 1), or the negative proves nothing.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::channel::Sender;
use smarm::cluster::discovery::{Discovery, Strategy};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::{start, Config, StaticSeeds, Timing};
use std::net::TcpListener;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
const ROLES: &[(&str, fn())] = &[("server", role_server), ("client", role_client)];
fn meta() -> NodeMeta {
NodeMeta {
role: "withdraw".into(),
region: "local".into(),
}
}
fn role_server() {
smarm::run(|| {
let cluster = start(Config {
node_name: "server".into(),
meta: meta(),
listen_addr: std::env::var("SMARM_LISTEN_ADDR")
.unwrap_or_else(|_| "127.0.0.1:0".into()),
strategy: Box::new(StaticSeeds::new(Vec::<(String, String)>::new())),
timing: Timing::default(),
})
.expect("binds");
println!("LISTENING {}", cluster.local_addr());
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// Scripted strategy: decoy, pause, withdraw decoy, real server, done.
struct Script {
decoy: String,
server: String,
}
impl Strategy for Script {
fn run(self: Box<Self>, out: Sender<Discovery>) {
let _ = out.send(Discovery::Candidate {
name: "ghost".into(),
addr: self.decoy.clone(),
});
// Long enough for the 250ms/500ms retries to land: ≥ 3 dials.
smarm::sleep(Duration::from_millis(1100));
let _ = out.send(Discovery::Withdrawn {
name: "ghost".into(),
addr: self.decoy,
});
let _ = out.send(Discovery::Candidate {
name: "server".into(),
addr: self.server,
});
}
}
fn role_client() {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
// The decoy: accept-and-close on an OS thread; count every accept.
let decoy = TcpListener::bind("127.0.0.1:0").unwrap();
let decoy_addr = decoy.local_addr().unwrap().to_string();
let dials = Arc::new(AtomicUsize::new(0));
let counter = dials.clone();
std::thread::spawn(move || {
for conn in decoy.incoming() {
counter.fetch_add(1, Ordering::SeqCst);
drop(conn);
}
});
smarm::run(move || {
let _cluster = start(Config {
node_name: "client".into(),
meta: meta(),
listen_addr: "127.0.0.1:0".into(),
strategy: Box::new(Script {
decoy: decoy_addr,
server: server_addr,
}),
timing: Timing::default(),
})
.expect("binds");
let ev = subscribe().unwrap();
loop {
match ev.rx.recv() {
Ok(NodeEvent::NodeUp(i)) if i.name == "server" => break,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
// The withdrawal preceded the server candidate in the strategy's
// stream, so it has been applied. Any dial that started before it
// is bounded by the connect+handshake deadlines; let it drain, then
// hold the count flat across a window longer than the pending
// backoff would be (1s at this point, 2s next).
let before = dials.load(Ordering::SeqCst);
smarm::sleep(Duration::from_millis(500));
let settled = dials.load(Ordering::SeqCst);
let t0 = Instant::now();
while t0.elapsed() < Duration::from_millis(3000) {
smarm::sleep(Duration::from_millis(100));
}
let after = dials.load(Ordering::SeqCst);
println!("WITHDRAWN before={before} settled={settled} after={after}");
println!("CLIENT DONE");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
#[test]
fn withdrawn_candidate_is_no_longer_dialed() {
maybe_child(ROLES);
let mut server = spawn_node("server", &[]);
let saddr = server.wait_listening();
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
let line = client.wait_line("WITHDRAWN", |l| l.starts_with("WITHDRAWN "));
let mut nums = line
.split_whitespace()
.skip(1)
.map(|kv| kv.split_once('=').unwrap().1.parse::<usize>().unwrap());
let (before, settled, after) = (
nums.next().unwrap(),
nums.next().unwrap(),
nums.next().unwrap(),
);
assert!(
before >= 1,
"decoy was never dialed; the negative proves nothing: {line}"
);
assert_eq!(
settled, after,
"connector kept dialing a withdrawn candidate: {line}"
);
client.wait_line("CLIENT DONE", |l| l == "CLIENT DONE");
}
+276
View File
@@ -0,0 +1,276 @@
//! RFC 010 c2 — owned envelope tests (roadmap: per-frame roundtrip,
//! truncation mid-field, unknown tag, length prefix lying long and short,
//! zero-length payload, adversarial lengths).
#![cfg(feature = "cluster")]
use serde::{Deserialize, Serialize};
use smarm::cluster::envelope::{
decode_payload, encode_payload, DecodeError, Frame, NodeMeta, RejectReason, MAX_FRAME_LEN,
PROTO_VERSION,
};
use smarm::cluster::RemoteDownReason;
use smarm::monitor::DownReason;
use smarm::pg::Incarnation;
fn meta() -> NodeMeta {
NodeMeta {
role: "worker".into(),
region: "eu-west".into(),
}
}
fn all_frames() -> Vec<Frame> {
vec![
Frame::Hello {
proto_version: PROTO_VERSION,
build_hash: 0xDEAD_BEEF_CAFE_F00D,
node_name: "alpha".into(),
incarnation: Incarnation::new(7),
meta: meta(),
},
Frame::HelloAck {
node_name: "beta".into(),
incarnation: Incarnation::new(9),
meta: meta(),
},
Frame::HelloReject {
reason: RejectReason::NameTaken,
},
Frame::Heartbeat,
Frame::Send {
index: 42,
generation: 3,
type_hash: 0x1234_5678_9ABC_DEF0,
payload: vec![1, 2, 3, 4, 5],
},
Frame::SendNamed {
name: "the_counter".into(),
type_hash: 0xFFFF_0000_FFFF_0000,
payload: vec![],
},
Frame::Monitor {
monitor_id: 77,
index: 42,
generation: 3,
},
Frame::Demonitor { monitor_id: 77 },
Frame::Down {
monitor_id: 77,
reason: RemoteDownReason::Local(DownReason::Panic),
},
Frame::Down {
monitor_id: 78,
reason: RemoteDownReason::Disconnected,
},
]
}
fn encode_one(f: &Frame) -> Vec<u8> {
let mut buf = Vec::new();
f.encode(&mut buf).unwrap();
buf
}
#[test]
fn per_frame_roundtrip() {
for f in all_frames() {
let buf = encode_one(&f);
let (decoded, consumed) = Frame::decode(&buf).unwrap().unwrap();
assert_eq!(decoded, f, "roundtrip mismatch");
assert_eq!(consumed, buf.len(), "consumed != buffer length for {f:?}");
}
}
#[test]
fn back_to_back_frames_decode_sequentially() {
let mut buf = Vec::new();
for f in all_frames() {
f.encode(&mut buf).unwrap();
}
let mut off = 0;
let mut decoded = Vec::new();
while off < buf.len() {
let (f, n) = Frame::decode(&buf[off..]).unwrap().unwrap();
decoded.push(f);
off += n;
}
assert_eq!(decoded, all_frames());
assert_eq!(off, buf.len());
}
#[test]
fn heartbeat_golden_bytes() {
// Locks the layout: u32 LE length prefix, then the tag byte.
let buf = encode_one(&Frame::Heartbeat);
assert_eq!(buf, vec![1, 0, 0, 0, 4]);
}
#[test]
fn zero_length_payload_roundtrips() {
let f = Frame::Send {
index: 0,
generation: 0,
type_hash: 0,
payload: vec![],
};
let buf = encode_one(&f);
let (decoded, consumed) = Frame::decode(&buf).unwrap().unwrap();
assert_eq!(decoded, f);
assert_eq!(consumed, buf.len());
}
#[test]
fn incomplete_is_none_not_error() {
let buf = encode_one(&all_frames()[0]);
// Every strict prefix short of the full frame must report "need more".
for cut in 0..buf.len() {
assert_eq!(
Frame::decode(&buf[..cut]).unwrap(),
None,
"cut at {cut} should be incomplete"
);
}
}
#[test]
fn unknown_frame_tag() {
let buf = vec![1, 0, 0, 0, 250];
assert_eq!(Frame::decode(&buf), Err(DecodeError::UnknownTag(250)));
}
#[test]
fn unknown_enum_tags() {
// HelloReject with a bogus reason tag.
let buf = vec![2, 0, 0, 0, 3, 99];
assert_eq!(
Frame::decode(&buf),
Err(DecodeError::UnknownEnumTag {
what: "RejectReason",
tag: 99
})
);
// Down with a bogus reason tag (id = 0u64).
let mut buf = vec![10, 0, 0, 0, 9];
buf.extend_from_slice(&0u64.to_le_bytes());
buf.push(200);
assert_eq!(
Frame::decode(&buf),
Err(DecodeError::UnknownEnumTag {
what: "RemoteDownReason",
tag: 200
})
);
}
#[test]
fn length_prefix_lying_long_with_bytes_present_is_trailing() {
let mut buf = encode_one(&Frame::Heartbeat);
// Declare 3 extra body bytes and actually supply them.
let declared = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) + 3;
buf[0..4].copy_from_slice(&declared.to_le_bytes());
buf.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
assert_eq!(Frame::decode(&buf), Err(DecodeError::Trailing { extra: 3 }));
}
#[test]
fn length_prefix_lying_long_without_bytes_is_incomplete() {
// Indistinguishable from a partial read — must be None, not an error.
let mut buf = encode_one(&Frame::Heartbeat);
let declared = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) + 3;
buf[0..4].copy_from_slice(&declared.to_le_bytes());
assert_eq!(Frame::decode(&buf).unwrap(), None);
}
#[test]
fn length_prefix_lying_short_truncates_a_field() {
let f = &all_frames()[0]; // Hello: plenty of fields to cut into
let mut buf = encode_one(f);
let declared = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]);
let lie = declared - 4; // cut mid-field
buf[0..4].copy_from_slice(&lie.to_le_bytes());
assert_eq!(Frame::decode(&buf), Err(DecodeError::Truncated));
}
#[test]
fn truncation_mid_string_field() {
// A frame whose declared length is intact but whose inner string length
// runs past the body: SendNamed claiming a 1000-byte name in a tiny body.
let mut body = vec![6u8]; // TAG_SEND_NAMED
body.extend_from_slice(&1000u16.to_le_bytes());
body.extend_from_slice(b"short");
let mut buf = (body.len() as u32).to_le_bytes().to_vec();
buf.extend_from_slice(&body);
assert_eq!(Frame::decode(&buf), Err(DecodeError::Truncated));
}
#[test]
fn adversarial_lengths() {
// Length prefix of u32::MAX: reject as oversized, do not wait for 4 GiB.
let buf = [0xFF, 0xFF, 0xFF, 0xFF, 0];
assert_eq!(
Frame::decode(&buf),
Err(DecodeError::FrameTooLarge {
declared: u32::MAX as usize
})
);
// Just over the cap: also rejected.
let over = (MAX_FRAME_LEN as u32 + 1).to_le_bytes();
assert!(matches!(
Frame::decode(&over),
Err(DecodeError::FrameTooLarge { .. })
));
// Zero-length frame: there is no tag byte; corrupt, not incomplete.
let buf = [0, 0, 0, 0];
assert_eq!(Frame::decode(&buf), Err(DecodeError::EmptyFrame));
}
#[test]
fn invalid_utf8_in_string_field() {
let mut buf = encode_one(&Frame::SendNamed {
name: "abcd".into(),
type_hash: 0,
payload: vec![],
});
// name bytes start after: 4 (len) + 1 (tag) + 2 (str len) = offset 7
buf[7] = 0xFF;
assert_eq!(Frame::decode(&buf), Err(DecodeError::Utf8));
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
struct Ping {
seq: u64,
label: String,
}
#[test]
fn payload_seam_roundtrip() {
let ping = Ping {
seq: 31337,
label: "hello".into(),
};
let blob = encode_payload(&ping).unwrap();
// Carry it through a real frame, as it will travel in c9.
let f = Frame::Send {
index: 1,
generation: 1,
type_hash: 0xABCD,
payload: blob,
};
let buf = encode_one(&f);
let (decoded, _) = Frame::decode(&buf).unwrap().unwrap();
let Frame::Send { payload, .. } = decoded else {
panic!("wrong frame");
};
let back: Ping = decode_payload(&payload).unwrap();
assert_eq!(back, ping);
}
#[test]
fn payload_seam_rejects_truncated_blob() {
let blob = encode_payload(&Ping {
seq: 1,
label: "x".into(),
})
.unwrap();
assert!(decode_payload::<Ping>(&blob[..blob.len() - 1]).is_err());
}
+161
View File
@@ -0,0 +1,161 @@
//! RFC 010 c8 — exposure registry + type hashing. Purely local, no network.
//!
//! Payload types are std types (`String`, `u64`) because the crate's serde is
//! deliberately derive-less (`default-features = false`) — user crates bring
//! their own derive; the contract here is `DeserializeOwned`.
//!
//! The hash-stability test re-execs the current binary (the c4 harness): the
//! guarantee under test is "stable across runs in the SAME binary" — exactly
//! what the build-hash handshake reduces the mesh to — not stability across
//! builds, which the scope guard explicitly rejects.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::encode_payload;
use smarm::cluster::expose::{
decode_deliver, decoder_registered, expose, expose_type, exposed_hash, exposed_names,
type_hash, DeliverError,
};
use smarm::monitor::{monitor, terminal_reason, DownReason};
use smarm::{channel, register, run, spawn, Name};
const ROLES: &[(&str, fn())] = &[("hasher", role_hasher)];
/// Print the hashes this process computes; the parent (a different run of
/// the same binary) compares against its own.
fn role_hasher() {
println!("HASH-STRING {}", type_hash::<String>());
println!("HASH-U64 {}", type_hash::<u64>());
}
const GREETER: Name<String> = Name::new("expose-test.greeter");
/// Exposed and unexposed lookup, the returned hash, and the audit listing.
#[test]
fn exposed_and_unexposed_lookup() {
maybe_child(ROLES);
run(|| {
let h = expose(GREETER);
assert_eq!(h, type_hash::<String>());
assert_eq!(exposed_hash("expose-test.greeter"), Some(h));
assert_eq!(exposed_hash("never-exposed"), None);
assert!(exposed_names().contains(&("expose-test.greeter", h)));
});
}
/// Distinct types land on distinct hashes (FNV over distinct TypeIds — a
/// smoke assertion; a collision would degrade to a decode error, never a
/// misroute, per RFC §3).
#[test]
fn distinct_types_distinct_hashes() {
maybe_child(ROLES);
run(|| {
assert_ne!(type_hash::<String>(), type_hash::<u64>());
assert_ne!(type_hash::<String>(), type_hash::<Vec<u8>>());
});
}
/// The decode-and-deliver contract: a registered hash decodes into the
/// target's typed channel; an unknown hash, corrupt bytes, and a missing
/// channel each fail without delivering — `WrongChannel`, never a misroute.
#[test]
fn decoder_registration_and_delivery() {
maybe_child(ROLES);
run(|| {
let h_string = expose_type::<String>();
let h_u64 = expose_type::<u64>();
assert!(decoder_registered(h_string));
assert!(!decoder_registered(h_string.wrapping_add(1)));
// A live actor with a String channel (registered from its own body,
// announced via a ready signal — the tests/registry.rs idiom).
let (ready_tx, ready_rx) = channel::<()>();
let (stop_tx, stop_rx) = channel::<()>();
let (msg_tx, msg_rx) = channel::<String>();
let pid = spawn(move || {
register(Name::<String>::new("expose-test.sink"), msg_tx).unwrap();
ready_tx.send(()).unwrap();
let _ = stop_rx.recv();
})
.pid();
ready_rx.recv().unwrap();
// Happy path: decode + deliver through the published channel.
let bytes = encode_payload("hello across the seam").unwrap();
decode_deliver(h_string, pid, &bytes).unwrap();
assert_eq!(msg_rx.recv().unwrap(), "hello across the seam");
// Unknown hash: nothing was registered under it.
assert!(matches!(
decode_deliver(h_string.wrapping_add(1), pid, &bytes),
Err(DeliverError::UnknownType)
));
// Corrupt bytes: the decoder fails before any send.
assert!(matches!(
decode_deliver(h_string, pid, &[0xff; 3]),
Err(DeliverError::Decode(_))
));
// Right decoder, wrong channel: the actor has no u64 channel, so the
// decoded value is refused — the NoChannel guarantee.
let u64_bytes = encode_payload(&7u64).unwrap();
assert!(matches!(
decode_deliver(h_u64, pid, &u64_bytes),
Err(DeliverError::WrongChannel)
));
stop_tx.send(()).unwrap();
});
}
/// `expose` and the bridge crossing agree on the resulting set: both funnel
/// the pid-boundary mark through the watchable machinery, so an exposed
/// name's holder dies with a terminal record — the exact observable
/// `mark_watchable` guarantees the membrane. (For named holders the mark is
/// already stamped by `register` itself; this pins the shared contract.)
#[test]
fn expose_and_bridge_crossing_agree_on_the_set() {
maybe_child(ROLES);
run(|| {
let (ready_tx, ready_rx) = channel::<()>();
let (stop_tx, stop_rx) = channel::<()>();
let (msg_tx, _msg_rx) = channel::<String>();
let pid = spawn(move || {
register(GREETER, msg_tx).unwrap();
ready_tx.send(()).unwrap();
let _ = stop_rx.recv();
})
.pid();
ready_rx.recv().unwrap();
expose(GREETER);
let m = monitor(pid);
stop_tx.send(()).unwrap();
assert_eq!(m.rx.recv().unwrap().reason, DownReason::Exit);
assert_eq!(terminal_reason(pid), Some(DownReason::Exit));
});
}
/// Hash stability across runs in the same binary: a re-exec of this binary
/// computes the same hashes this process does.
#[test]
fn hash_stable_across_runs_in_same_binary() {
maybe_child(ROLES);
let (mine_string, mine_u64) = {
// Computing a TypeId hash needs no runtime, but keep the contract
// uniform with real call sites.
(type_hash::<String>(), type_hash::<u64>())
};
let mut child = spawn_node("hasher", &[]);
let line = child.wait_line("HASH-STRING", |l| l.starts_with("HASH-STRING "));
assert_eq!(
line["HASH-STRING ".len()..].parse::<u64>().unwrap(),
mine_string
);
let line = child.wait_line("HASH-U64", |l| l.starts_with("HASH-U64 "));
assert_eq!(line["HASH-U64 ".len()..].parse::<u64>().unwrap(), mine_u64);
child.wait_exit();
}
+240
View File
@@ -0,0 +1,240 @@
//! RFC 010 c5 — handshake state-machine tests (roadmap: happy path; hash
//! mismatch; proto-version mismatch; name already claimed; simultaneous-connect
//! tie-break; garbage before Hello). Pure — no IO, no actors, no runtime.
#![cfg(feature = "cluster")]
use smarm::cluster::envelope::{Frame, NodeMeta, RejectReason, PROTO_VERSION};
use smarm::cluster::handshake::{
dial_wins, Initiator, InitiatorOutcome, Local, PeerStanding, Responder, ResponderOutcome,
};
use smarm::pg::Incarnation;
const HASH: u64 = 0xDEAD_BEEF_CAFE_F00D;
fn local(name: &str) -> Local {
Local {
node_name: name.into(),
incarnation: Incarnation::new(7),
build_hash: HASH,
meta: NodeMeta {
role: "worker".into(),
region: "eu-west".into(),
},
}
}
/// The Hello that `Initiator::new(&local(name))` emits, built by hand.
fn hello_from(name: &str) -> Frame {
let l = local(name);
Frame::Hello {
proto_version: PROTO_VERSION,
build_hash: l.build_hash,
node_name: l.node_name,
incarnation: l.incarnation,
meta: l.meta,
}
}
#[test]
fn happy_path_establishes_both_ends() {
// alpha dials beta.
let (initiator, hello) = Initiator::new(&local("alpha"));
assert_eq!(hello, hello_from("alpha"), "initiator emits its identity");
let responder = Responder::new(local("beta"));
let (reply, peer) = match responder.on_frame(hello, PeerStanding::Free) {
ResponderOutcome::Accepted { reply, peer } => (reply, peer),
other => panic!("expected Accepted, got {other:?}"),
};
assert_eq!(peer.node_name, "alpha");
assert_eq!(peer.incarnation, Incarnation::new(7));
assert_eq!(peer.meta.role, "worker");
// The ack carries the responder's identity, no hash/version (one-sided
// check — sound because equality is symmetric).
let l = local("beta");
assert_eq!(
reply,
Frame::HelloAck {
node_name: l.node_name,
incarnation: l.incarnation,
meta: l.meta,
}
);
match initiator.on_frame(reply) {
InitiatorOutcome::Established(peer) => {
assert_eq!(peer.node_name, "beta");
assert_eq!(peer.incarnation, Incarnation::new(7));
assert_eq!(peer.meta.region, "eu-west");
}
other => panic!("expected Established, got {other:?}"),
}
}
#[test]
fn hash_mismatch_rejected() {
let responder = Responder::new(local("beta"));
let hello = Frame::Hello {
proto_version: PROTO_VERSION,
build_hash: HASH ^ 1,
node_name: "alpha".into(),
incarnation: Incarnation::new(7),
meta: local("alpha").meta,
};
match responder.on_frame(hello, PeerStanding::Free) {
ResponderOutcome::Rejected { reply, reason } => {
assert_eq!(reason, RejectReason::HashMismatch);
assert_eq!(reply, Frame::HelloReject { reason });
}
other => panic!("expected Rejected, got {other:?}"),
}
// The dialer side of the same story: a reject frame comes back.
let (initiator, _hello) = Initiator::new(&local("alpha"));
match initiator.on_frame(Frame::HelloReject {
reason: RejectReason::HashMismatch,
}) {
InitiatorOutcome::Rejected(RejectReason::HashMismatch) => {}
other => panic!("expected Rejected(HashMismatch), got {other:?}"),
}
}
#[test]
fn proto_version_mismatch_rejected_and_checked_first() {
// Both proto and hash wrong: proto wins — nothing after the version can
// be trusted, and HelloReject is the cross-version compatibility anchor.
let responder = Responder::new(local("beta"));
let hello = Frame::Hello {
proto_version: PROTO_VERSION + 1,
build_hash: HASH ^ 1,
node_name: "alpha".into(),
incarnation: Incarnation::new(7),
meta: local("alpha").meta,
};
match responder.on_frame(hello, PeerStanding::Free) {
ResponderOutcome::Rejected { reason, .. } => {
assert_eq!(reason, RejectReason::ProtoVersion);
}
other => panic!("expected Rejected, got {other:?}"),
}
}
#[test]
fn claimed_name_rejected() {
let responder = Responder::new(local("beta"));
let ctx = PeerStanding::Claimed;
match responder.on_frame(hello_from("alpha"), ctx) {
ResponderOutcome::Rejected { reason, .. } => {
assert_eq!(reason, RejectReason::NameTaken);
}
other => panic!("expected Rejected, got {other:?}"),
}
}
#[test]
fn own_name_offered_rejected_as_name_taken() {
// Self-connect or genuine collision: the responder's own name arrives.
let responder = Responder::new(local("beta"));
match responder.on_frame(hello_from("beta"), PeerStanding::Free) {
ResponderOutcome::Rejected { reason, .. } => {
assert_eq!(reason, RejectReason::NameTaken);
}
other => panic!("expected Rejected, got {other:?}"),
}
}
#[test]
fn hash_checked_before_name() {
// Wrong hash AND claimed name: hash wins (validity before identity).
let responder = Responder::new(local("beta"));
let hello = Frame::Hello {
proto_version: PROTO_VERSION,
build_hash: HASH ^ 1,
node_name: "alpha".into(),
incarnation: Incarnation::new(7),
meta: local("alpha").meta,
};
let ctx = PeerStanding::Claimed;
match responder.on_frame(hello, ctx) {
ResponderOutcome::Rejected { reason, .. } => {
assert_eq!(reason, RejectReason::HashMismatch);
}
other => panic!("expected Rejected, got {other:?}"),
}
}
#[test]
fn dial_wins_is_deterministic_and_antisymmetric() {
// The smaller name's dial survives; both ends compute the same verdict.
assert!(dial_wins("alpha", "beta"));
assert!(!dial_wins("beta", "alpha"));
for (a, b) in [("a", "b"), ("node-1", "node-2"), ("x", "xx")] {
assert_ne!(dial_wins(a, b), dial_wins(b, a), "({a}, {b})");
}
}
#[test]
fn simultaneous_connect_exactly_one_side_accepts() {
// alpha and beta dial each other at once. Each responder sees the peer's
// Hello while its own dial is in flight.
let ctx = PeerStanding::Dialing;
// On beta: inbound is alpha's dial; alpha < beta, so the inbound wins.
let on_beta = Responder::new(local("beta")).on_frame(hello_from("alpha"), ctx);
assert!(
matches!(on_beta, ResponderOutcome::Accepted { .. }),
"beta must accept alpha's dial, got {on_beta:?}"
);
// On alpha: inbound is beta's dial; it loses — close silently, no frame
// (ratified: both ends can compute the outcome, a reject adds nothing).
let on_alpha = Responder::new(local("alpha")).on_frame(hello_from("beta"), ctx);
assert!(
matches!(on_alpha, ResponderOutcome::TieBreakLoss),
"alpha must silently drop beta's dial, got {on_alpha:?}"
);
}
#[test]
fn tiebreak_loss_only_applies_when_dialing() {
// Same inbound Hello, no dial in flight: plain accept.
let on_alpha = Responder::new(local("alpha")).on_frame(hello_from("beta"), PeerStanding::Free);
assert!(matches!(on_alpha, ResponderOutcome::Accepted { .. }));
}
#[test]
fn garbage_before_hello_fails_without_reply() {
// Any valid-but-wrong frame before Hello is a protocol violation: close,
// no reject frame. (Undecodable bytes are the codec's Err, not ours.)
for frame in [
Frame::Heartbeat,
Frame::HelloAck {
node_name: "alpha".into(),
incarnation: Incarnation::new(7),
meta: local("alpha").meta,
},
Frame::Demonitor { monitor_id: 3 },
] {
let out = Responder::new(local("beta")).on_frame(frame.clone(), PeerStanding::Free);
match out {
ResponderOutcome::Failed(f) => assert_eq!(f, frame),
other => panic!("expected Failed({frame:?}), got {other:?}"),
}
}
}
#[test]
fn garbage_before_ack_fails_the_initiator() {
for frame in [
Frame::Heartbeat,
hello_from("beta"),
Frame::Demonitor { monitor_id: 3 },
] {
let (initiator, _hello) = Initiator::new(&local("alpha"));
match initiator.on_frame(frame.clone()) {
InitiatorOutcome::Failed(f) => assert_eq!(f, frame),
other => panic!("expected Failed({frame:?}), got {other:?}"),
}
}
}
+270
View File
@@ -0,0 +1,270 @@
//! RFC 010 c7a — membership events and the view, at the manager.
//!
//! Same construction as the c6a lifecycle suite: the handshake is bypassed,
//! connections are built already-established over localhost TCP pairs with
//! fabricated `Peer`s, and the manager is started plainly so the test can
//! terminate. What is under test is the membership layer that c7 adds to the
//! manager: `node_up`/`node_down` events to subscribers (snapshot-then-stream),
//! the view, and NodeId identity — memoized per `(name, incarnation)`, so a
//! reconnect blip keeps its id and a restart (new incarnation) gets a fresh one.
#![cfg(feature = "cluster")]
use std::time::Duration;
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::handshake::Peer;
use smarm::cluster::manager::{Call, Manager, Reply, MANAGER};
use smarm::cluster::membership::{subscribe, view, MembershipEvents, NodeEvent};
use smarm::cluster::spawn_established;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{Conn, FramedConn, Transport};
use smarm::cluster::Timing;
use smarm::gen_server::{self, GenServerBuilder};
use smarm::pg::{Incarnation, NodeId};
use smarm::run;
/// A fabricated post-handshake peer identity, with the incarnation under the
/// test's control (it is identity-bearing here, unlike in the c6a suite).
fn peer(name: &str, inc: u32) -> Peer {
Peer {
node_name: name.to_string(),
incarnation: Incarnation::new(inc),
meta: NodeMeta {
role: "test".to_string(),
region: "test".to_string(),
},
}
}
/// One established transport pair over localhost (TCP backlog covers the
/// sequential dial-then-accept, as in the c3 conformance suite).
fn pair(t: &dyn Transport) -> (Box<dyn Conn>, Box<dyn Conn>) {
let mut l = t.listen("127.0.0.1:0").unwrap();
let a = t.dial(&l.local_addr()).unwrap();
let b = l.accept().unwrap();
(a, b)
}
/// The next event, or a panic naming the wait. The bound is generous against
/// a sub-millisecond real cost.
fn next_event(ev: &MembershipEvents, waiting_for: &str) -> NodeEvent {
ev.rx
.recv_timeout(Duration::from_secs(5))
.unwrap_or_else(|e| panic!("timed out waiting for {waiting_for}: {e:?}"))
}
/// Assert the subscription is drained: no event is pending.
fn assert_quiet(ev: &MembershipEvents) {
assert!(matches!(ev.rx.try_recv(), Ok(None)));
}
fn disconnect(name: &str) {
assert!(matches!(
gen_server::call(
MANAGER,
Call::Disconnect {
name: name.to_string()
}
),
Ok(Reply::Disconnected)
));
}
/// Live subscription: an empty snapshot, then `NodeUp` on registration and
/// `NodeDown` (same id) on commanded disconnect and on peer EOF alike.
#[test]
fn subscriber_sees_up_and_down() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let ev = subscribe().expect("manager is up");
assert_quiet(&ev); // nothing live: the snapshot is empty
let t = TcpTransport;
let (a1, b1) = pair(&t);
let (a2, b2) = pair(&t);
spawn_established(FramedConn::new(a1), peer("node-b", 1), Timing::default())
.expect("node-b registers");
spawn_established(FramedConn::new(a2), peer("node-c", 1), Timing::default())
.expect("node-c registers");
let up_b = match next_event(&ev, "node_up(node-b)") {
NodeEvent::NodeUp(info) => {
assert_eq!(info.name, "node-b");
assert_eq!(info.incarnation, Incarnation::new(1));
assert_eq!(info.meta.role, "test");
info
}
other => panic!("expected node_up(node-b), got {other:?}"),
};
let up_c = match next_event(&ev, "node_up(node-c)") {
NodeEvent::NodeUp(info) => {
assert_eq!(info.name, "node-c");
info
}
other => panic!("expected node_up(node-c), got {other:?}"),
};
assert_ne!(up_b.node, up_c.node, "distinct peers get distinct ids");
// Commanded disconnect: down with node-b's id.
disconnect("node-b");
assert_eq!(
next_event(&ev, "node_down(node-b)"),
NodeEvent::NodeDown(up_b.clone())
);
// Peer EOF, no command: down with node-c's id.
drop(b2);
assert_eq!(
next_event(&ev, "node_down(node-c)"),
NodeEvent::NodeDown(up_c.clone())
);
assert_quiet(&ev);
drop(b1);
mgr.shutdown();
});
}
/// Snapshot-then-stream: a subscriber arriving after connections established
/// receives one `NodeUp` per live peer before anything else, and the view
/// call agrees with it.
#[test]
fn late_subscriber_gets_snapshot_and_view_agrees() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let t = TcpTransport;
let (a1, b1) = pair(&t);
let (a2, b2) = pair(&t);
spawn_established(FramedConn::new(a1), peer("node-b", 1), Timing::default())
.expect("node-b registers");
spawn_established(FramedConn::new(a2), peer("node-c", 1), Timing::default())
.expect("node-c registers");
let ev = subscribe().expect("manager is up");
let mut names = Vec::new();
for _ in 0..2 {
match next_event(&ev, "a snapshot node_up") {
NodeEvent::NodeUp(info) => names.push(info.name),
other => panic!("expected a snapshot node_up, got {other:?}"),
}
}
names.sort();
assert_eq!(names, ["node-b", "node-c"]);
assert_quiet(&ev); // the snapshot is exactly the live set
let mut v = view().expect("manager is up");
v.sort_by(|a, b| a.name.cmp(&b.name));
assert_eq!(v.len(), 2);
assert_eq!(v[0].name, "node-b");
assert_eq!(v[1].name, "node-c");
disconnect("node-b");
disconnect("node-c");
drop((b1, b2));
// Drain the two downs so the subscription ends quiet.
let _ = next_event(&ev, "node_down");
let _ = next_event(&ev, "node_down");
mgr.shutdown();
});
}
/// NodeId identity: a restart (same name, new incarnation) is a NEW id — the
/// ghost and its successor are distinguishable — while a reconnect blip (same
/// name, same incarnation) keeps its id.
#[test]
fn restart_gets_new_id_blip_keeps_id() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let ev = subscribe().expect("manager is up");
let t = TcpTransport;
let id = |e: NodeEvent, what: &str| -> NodeId {
match e {
NodeEvent::NodeUp(info) => info.node,
other => panic!("expected node_up ({what}), got {other:?}"),
}
};
let down_id = |e: NodeEvent, what: &str| -> NodeId {
match e {
NodeEvent::NodeDown(info) => info.node,
other => panic!("expected node_down ({what}), got {other:?}"),
}
};
// Up at incarnation 1, then the peer dies (EOF).
let (a1, b1) = pair(&t);
spawn_established(FramedConn::new(a1), peer("node-b", 1), Timing::default())
.expect("registers");
let id1 = id(next_event(&ev, "node_up inc 1"), "inc 1");
drop(b1);
assert_eq!(down_id(next_event(&ev, "node_down inc 1"), "inc 1"), id1);
// Restart: new incarnation, new id — the ghost's id is not reused.
let (a2, b2) = pair(&t);
spawn_established(FramedConn::new(a2), peer("node-b", 2), Timing::default())
.expect("registers");
let id2 = id(next_event(&ev, "node_up inc 2"), "inc 2");
assert_ne!(
id1, id2,
"a restarted node must be distinguishable from its ghost"
);
// Blip: the same incarnation reconnects and keeps its id.
disconnect("node-b");
assert_eq!(down_id(next_event(&ev, "node_down inc 2"), "inc 2"), id2);
let (a3, b3) = pair(&t);
spawn_established(FramedConn::new(a3), peer("node-b", 2), Timing::default())
.expect("registers");
let id3 = id(next_event(&ev, "node_up after blip"), "blip");
assert_eq!(
id2, id3,
"a reconnect at the same incarnation is the same node"
);
disconnect("node-b");
let _ = next_event(&ev, "final node_down");
drop((b2, b3));
mgr.shutdown();
});
}
/// A dropped subscriber is pruned on the next emit and never disturbs the
/// manager or a live subscriber.
#[test]
fn dead_subscriber_is_pruned() {
run(|| {
let mgr = GenServerBuilder::new(Manager::new())
.named(MANAGER)
.start()
.expect("manager name is free");
let dead = subscribe().expect("manager is up");
drop(dead);
let live = subscribe().expect("manager is up");
let t = TcpTransport;
let (a1, b1) = pair(&t);
spawn_established(FramedConn::new(a1), peer("node-b", 1), Timing::default())
.expect("registers");
match next_event(&live, "node_up despite a dead co-subscriber") {
NodeEvent::NodeUp(info) => assert_eq!(info.name, "node-b"),
other => panic!("expected node_up, got {other:?}"),
}
disconnect("node-b");
let _ = next_event(&live, "node_down");
drop(b1);
mgr.shutdown();
});
}
+175
View File
@@ -0,0 +1,175 @@
//! RFC 010 c7 — the Phase 2 gate: a 3-node mesh under the subprocess
//! harness, repeatable.
//!
//! Each node process runs the integrated `cluster::start` (manager +
//! acceptor + connector + static seeds), subscribes to membership like any
//! consumer, and announces protocol-visible facts as lines:
//! `LISTENING <addr>`, `MEMBER-UP <name> inc=<n>`, `MEMBER-DOWN <name>`.
//! Then it **parks forever** — cross-process teardown is retractable state
//! (binding trap), so the parent SIGKILLs via `Node`'s `Drop` and clean exit
//! stays the c4 harness's own smoke test.
//!
//! Ports: nodes bind `:0` and report, so the mesh is built by seeding each
//! node with the previously-reported addresses (n1: no seeds; n2: n1;
//! n3: n1+n2 — inbound covers the reverse edges). The late-seed test is the
//! one exception: the parent pre-reserves a port by binding-and-closing it,
//! seeds one node with it, then starts the second node on that exact
//! address. In principle another process could steal the port in the gap;
//! in practice the window is microseconds on a local runner — accepted, and
//! confined to that one test.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node, Node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::{start, Config, StaticSeeds, Timing};
use std::time::Duration;
const ROLES: &[(&str, fn())] = &[("node", role_node)];
/// A mesh node: identity and seeds from env, membership events to stdout,
/// park forever (the parent reaps).
fn role_node() {
let name = std::env::var("SMARM_NODE_NAME").expect("SMARM_NODE_NAME not set");
let listen = std::env::var("SMARM_LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1:0".to_string());
// Seeds: comma-separated `name=addr` pairs; empty or unset means none.
let seeds: Vec<(String, String)> = std::env::var("SMARM_SEEDS")
.unwrap_or_default()
.split(',')
.filter(|s| !s.is_empty())
.map(|s| {
let (n, a) = s.split_once('=').expect("seed must be name=addr");
(n.to_string(), a.to_string())
})
.collect();
smarm::run(move || {
let cluster = start(Config {
node_name: name,
meta: NodeMeta {
role: "mesh-test".to_string(),
region: "local".to_string(),
},
listen_addr: listen,
strategy: Box::new(StaticSeeds::new(seeds)),
timing: Timing::default(),
})
.expect("listener binds");
println!("LISTENING {}", cluster.local_addr());
let events = subscribe().expect("manager is up");
loop {
match events.rx.recv() {
Ok(NodeEvent::NodeUp(info)) => {
println!("MEMBER-UP {} inc={}", info.name, info.incarnation.get());
}
Ok(NodeEvent::NodeDown(info)) => {
println!("MEMBER-DOWN {}", info.name);
}
Err(_) => break, // manager gone; park below regardless
}
}
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
fn spawn_mesh_node(name: &str, seeds: &str, listen: Option<&str>) -> Node {
let mut env: Vec<(&str, &str)> = vec![("SMARM_NODE_NAME", name), ("SMARM_SEEDS", seeds)];
if let Some(addr) = listen {
env.push(("SMARM_LISTEN_ADDR", addr));
}
spawn_node("node", &env)
}
/// Wait for `MEMBER-UP <peer> inc=<n>` and return the incarnation.
fn wait_member_up(node: &mut Node, peer: &str) -> u32 {
let prefix = format!("MEMBER-UP {peer} inc=");
let line = node.wait_line(&format!("MEMBER-UP {peer}"), |l| l.starts_with(&prefix));
line[prefix.len()..].parse().expect("incarnation parses")
}
fn wait_member_down(node: &mut Node, peer: &str) {
let want = format!("MEMBER-DOWN {peer}");
node.wait_line(&want, |l| l == want);
}
/// The gate, plus the kill and restart facts, as one mesh's life: three
/// nodes form a full mesh (every node sees both others up); killing one
/// yields `node_down` at both survivors; its restart under the same name
/// arrives as a NEW incarnation — the ghost and its successor are
/// distinguishable at every observer.
#[test]
fn three_node_mesh_forms_then_kill_then_restart_distinguishable() {
maybe_child(ROLES);
let mut n1 = spawn_mesh_node("node-1", "", None);
let a1 = n1.wait_listening();
let mut n2 = spawn_mesh_node("node-2", &format!("node-1={a1}"), None);
let a2 = n2.wait_listening();
let mut n3 = spawn_mesh_node("node-3", &format!("node-1={a1},node-2={a2}"), None);
let _a3 = n3.wait_listening();
// Full mesh: each node reports both peers up (dialed or inbound alike).
wait_member_up(&mut n1, "node-2");
let inc3_at_n1 = wait_member_up(&mut n1, "node-3");
wait_member_up(&mut n2, "node-1");
let inc3_at_n2 = wait_member_up(&mut n2, "node-3");
wait_member_up(&mut n3, "node-1");
wait_member_up(&mut n3, "node-2");
assert_eq!(
inc3_at_n1, inc3_at_n2,
"one node, one incarnation, all observers"
);
// Kill node-3 (SIGKILL via Drop): node_down at both survivors.
drop(n3);
wait_member_down(&mut n1, "node-3");
wait_member_down(&mut n2, "node-3");
// Restart node-3 under the same name: it re-dials its seeds and comes
// up everywhere as a new incarnation — never the ghost's.
let mut n3b = spawn_mesh_node("node-3", &format!("node-1={a1},node-2={a2}"), None);
let _ = n3b.wait_listening();
let inc3b_at_n1 = wait_member_up(&mut n1, "node-3");
let inc3b_at_n2 = wait_member_up(&mut n2, "node-3");
assert_eq!(inc3b_at_n1, inc3b_at_n2);
assert_ne!(
inc3_at_n1, inc3b_at_n1,
"a restarted node must be distinguishable from its ghost"
);
wait_member_up(&mut n3b, "node-1");
wait_member_up(&mut n3b, "node-2");
}
/// A seed that is unreachable at start is not fatal: the connector retries
/// on backoff, and when a node finally appears at that address, the mesh
/// edge forms.
#[test]
fn seed_unreachable_at_start_then_arriving_later() {
maybe_child(ROLES);
// Pre-reserve an address by binding and immediately closing it (see the
// module docs for the accepted steal window). Dials to it are refused
// until node-b starts there.
let reserved = {
let l = std::net::TcpListener::bind("127.0.0.1:0").expect("bind");
l.local_addr().expect("addr").to_string()
};
let mut a = spawn_mesh_node("node-a", &format!("node-b={reserved}"), None);
let _ = a.wait_listening();
// Let a few refused attempts happen before the seed comes up, so the
// retry path is what forms the edge (backoff cap 5s < harness WAIT 10s).
std::thread::sleep(Duration::from_millis(600));
let mut b = spawn_mesh_node("node-b", "", Some(&reserved));
let _ = b.wait_listening();
wait_member_up(&mut a, "node-b");
wait_member_up(&mut b, "node-a");
}
+359
View File
@@ -0,0 +1,359 @@
//! RFC 010 c12 — remote monitors.
//!
//! Local suite (`run()`, no network): the immediate answers — no connection
//! ⇒ `Disconnected`, dead incarnation ⇒ `NoProc` — and the self-node
//! collapse (a plain local monitor underneath, incl. `demonitor_remote`).
//!
//! Cross-process: a *server* exposes a control name and spawns workers on
//! request, replying with each worker's pid (via `RemotePid::from_local`,
//! the D12 set-site) or, for the deliberately unshipped one, only its raw
//! slot numbers. The *client* monitors them and asserts: kill ⇒ the true
//! reason (Exit / Panic); a corpse ⇒ its recorded terminal reason, not
//! NoProc; a live pid that never crossed the wire ⇒ NoProc (no liveness
//! leak); a demonitor racing the kill ⇒ no notice, proven by stream ORDER
//! (a later notice on the same connection arrives while the earlier slot
//! is still empty), not by sleeping.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::expose::{expose, expose_type};
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::remote::{
self, demonitor_remote, monitor_remote, send_to_remote, RemoteName, RemotePid,
};
use smarm::cluster::{start, Config, RemoteDownReason, StaticSeeds, Timing};
use smarm::pg::Incarnation;
use smarm::{channel, install, register, run, spawn, Addressable, DownReason, Erased, Name, Pid};
use std::collections::HashMap;
use std::time::Duration;
// ---- message types (hand-rolled serde; the crate is derive-less) ---------
#[derive(Debug)]
struct Ctl {
cmd: String,
reply_to: RemotePid<Client>,
}
#[derive(Debug)]
struct Answer {
text: String,
pid: Option<RemotePid<Erased>>,
}
struct Client;
impl Addressable for Client {
type Msg = Answer;
}
impl serde::Serialize for Ctl {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(2)?;
t.serialize_element(&self.cmd)?;
t.serialize_element(&self.reply_to)?;
t.end()
}
}
impl<'de> serde::Deserialize<'de> for Ctl {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (cmd, reply_to) = <(String, RemotePid<Client>)>::deserialize(d)?;
Ok(Ctl { cmd, reply_to })
}
}
impl serde::Serialize for Answer {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(2)?;
t.serialize_element(&self.text)?;
t.serialize_element(&self.pid)?;
t.end()
}
}
impl<'de> serde::Deserialize<'de> for Answer {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (text, pid) = <(String, Option<RemotePid<Erased>>)>::deserialize(d)?;
Ok(Answer { text, pid })
}
}
// ================= local suite =========================================
/// No connection to the pid's node: `Disconnected` at once — the remote
/// analog of NoProc, and the first thing c11's variant is for.
#[test]
fn unconnected_node_is_disconnected_immediately() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let ghost = RemotePid::<Erased>::from_parts("nowhere", Incarnation::new(1), 3, 1);
let m = monitor_remote(ghost.clone());
let d = m.recv().unwrap();
assert_eq!(d.pid, ghost);
assert_eq!(d.reason, RemoteDownReason::Disconnected);
});
}
/// The node is connected but the pid names an earlier incarnation: the
/// actor is a known corpse (RFC v2 §3), so `NoProc` at once — never
/// `Disconnected`, nothing on the wire.
#[test]
fn dead_incarnation_is_noproc_immediately() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (probe_tx, probe_rx) = channel();
remote::bind_outbound_probe("peer", Incarnation::new(5), probe_tx);
let stale = RemotePid::<Erased>::from_parts("peer", Incarnation::new(4), 9, 1);
let m = monitor_remote(stale);
assert_eq!(m.recv().unwrap().reason, DownReason::NoProc.into());
assert!(probe_rx.try_recv().unwrap().is_none(), "no frame emitted");
});
}
/// A self-node pid collapses to an ordinary local monitor: the true reason
/// on exit, and `demonitor_remote` cancels it.
#[test]
fn self_node_pid_collapses_to_local_monitor() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (go_tx, go_rx) = channel::<()>();
let (go2_tx, go2_rx) = channel::<()>();
let a = spawn(move || {
let _ = go_rx.recv();
})
.pid();
let b = spawn(move || {
let _ = go2_rx.recv();
})
.pid();
let ma = monitor_remote(RemotePid::from_local(a).expect("identity set"));
let mb = monitor_remote(RemotePid::from_local(b).expect("identity set"));
assert_ne!(ma.id, mb.id);
assert!(ma.target.local() == Some(a));
demonitor_remote(&mb);
go2_tx.send(()).unwrap();
go_tx.send(()).unwrap();
let d = ma.recv().unwrap();
assert_eq!(d.reason, DownReason::Exit.into());
assert_eq!(d.pid.local(), Some(a));
// `a` is down (its notice arrived), and `b` was killed first on the
// same scheduler — a notice for `b` would be here by now. After a
// demonitor the channel is closed-empty (`Err`), like the local one.
assert!(matches!(mb.try_recv(), Ok(None) | Err(_)));
});
}
// ================= cross-process ======================================
const ROLES: &[(&str, fn())] = &[("server", role_server), ("client", role_client)];
const CTL: Name<Ctl> = Name::new("c12.ctl");
fn cfg(name: &str, seeds: Vec<(String, String)>) -> Config {
Config {
node_name: name.to_string(),
meta: NodeMeta {
role: "c12".into(),
region: "local".into(),
},
listen_addr: std::env::var("SMARM_LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1:0".into()),
strategy: Box::new(StaticSeeds::new(seeds)),
timing: Timing::default(),
}
}
fn wait_up(events: &smarm::cluster::membership::MembershipEvents, who: &str) {
loop {
match events.rx.recv() {
Ok(NodeEvent::NodeUp(i)) if i.name == who => return,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
}
/// Server commands (all answered to `reply_to`):
/// - `spawn:exit` / `spawn:panic` — a parked worker; `kill:<index>` releases
/// it, whereupon it returns / panics. Answer carries its pid.
/// - `spawn:corpse` — a worker that has already exited when the answer is
/// sent; the pid was shipped (watchable) before it died.
/// - `spawn:unwatched` — a parked worker whose pid is NEVER shipped; the
/// answer carries only `text = "slot:<index>:<generation>"`.
fn role_server() {
smarm::run(move || {
let cluster = start(cfg("server", vec![])).expect("binds");
println!("LISTENING {}", cluster.local_addr());
let (tx, rx) = channel::<Ctl>();
register(CTL, tx).unwrap();
expose(CTL);
println!("READY");
let mut workers: HashMap<u32, smarm::channel::Sender<()>> = HashMap::new();
loop {
let ctl = rx.recv().unwrap();
println!("CTL {}", ctl.cmd);
let (text, pid): (String, Option<RemotePid<Erased>>) = match ctl.cmd.as_str() {
"spawn:exit" | "spawn:panic" => {
let panic = ctl.cmd == "spawn:panic";
let (go_tx, go_rx) = channel::<()>();
let p: Pid = spawn(move || {
let _ = go_rx.recv();
if panic {
panic!("worker asked to panic");
}
})
.pid();
workers.insert(p.index(), go_tx);
(
"ok".into(),
Some(RemotePid::from_local(p).expect("identity set")),
)
}
"spawn:corpse" => {
let p: Pid = spawn(|| {}).pid();
let rp = RemotePid::from_local(p).expect("identity set"); // shipped ⇒ watchable
let m = smarm::monitor(p);
let _ = m.rx.recv(); // dead before the answer goes out
("ok".into(), Some(rp))
}
"spawn:unwatched" => {
let (go_tx, go_rx) = channel::<()>();
let p: Pid = spawn(move || {
let _ = go_rx.recv();
})
.pid();
workers.insert(p.index(), go_tx);
(format!("slot:{}:{}", p.index(), p.generation()), None)
}
other => {
let idx: u32 = other.strip_prefix("kill:").unwrap().parse().unwrap();
if let Some(go) = workers.remove(&idx) {
let _ = go.send(());
}
("killed".into(), None)
}
};
send_to_remote(ctl.reply_to, Answer { text, pid }).unwrap();
}
});
}
fn role_client() {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
smarm::run(move || {
let _cluster = start(cfg("client", vec![("server".into(), server_addr)])).expect("binds");
let ev = subscribe().unwrap();
wait_up(&ev, "server");
let (tx, rx) = channel::<Answer>();
let me: Pid<Client> = install::<Client>(tx);
expose_type::<Answer>();
let ask = |cmd: &str| -> Answer {
remote::send(
RemoteName::new("server", CTL),
Ctl {
cmd: cmd.into(),
reply_to: RemotePid::from_local(me).expect("identity set"),
},
)
.unwrap();
rx.recv().unwrap()
};
let server_inc = ev_incarnation();
// 1. kill ⇒ true reason (Exit).
let a = ask("spawn:exit").pid.unwrap();
let ma = monitor_remote(a.clone());
ask(&format!("kill:{}", a.index()));
let d = ma.recv().unwrap();
assert_eq!(d.pid, a);
println!("DOWN exit {:?}", d.reason);
// 2. kill ⇒ true reason (Panic).
let b = ask("spawn:panic").pid.unwrap();
let mb = monitor_remote(b.clone());
ask(&format!("kill:{}", b.index()));
println!("DOWN panic {:?}", mb.recv().unwrap().reason);
// 3. corpse ⇒ recorded terminal reason, not NoProc.
let c = ask("spawn:corpse").pid.unwrap();
println!("DOWN corpse {:?}", monitor_remote(c).recv().unwrap().reason);
// 4. live but never shipped/exposed ⇒ NoProc (no leak); a made-up
// slot on the same node ⇒ NoProc too, indistinguishably.
let ans = ask("spawn:unwatched");
let mut it = ans.text.strip_prefix("slot:").unwrap().split(':');
let (idx, gen): (u32, u32) = (
it.next().unwrap().parse().unwrap(),
it.next().unwrap().parse().unwrap(),
);
let hidden = RemotePid::<Erased>::from_parts("server", server_inc, idx, gen);
println!(
"DOWN hidden {:?}",
monitor_remote(hidden).recv().unwrap().reason
);
let bogus = RemotePid::<Erased>::from_parts("server", server_inc, 100_000, 1);
println!(
"DOWN bogus {:?}",
monitor_remote(bogus).recv().unwrap().reason
);
// 5. demonitor races the kill: no notice for `d1`, proven by order —
// `d2`'s notice (same connection, later) arrives while `d1`'s
// slot is still empty.
let d1 = ask("spawn:exit").pid.unwrap();
let m1 = monitor_remote(d1.clone());
demonitor_remote(&m1);
ask(&format!("kill:{}", d1.index()));
let d2 = ask("spawn:exit").pid.unwrap();
let m2 = monitor_remote(d2.clone());
ask(&format!("kill:{}", d2.index()));
assert_eq!(m2.recv().unwrap().reason, DownReason::Exit.into());
// Closed-empty (`Err`) or open-empty (`Ok(None)`) both mean no notice.
let stray = matches!(m1.try_recv(), Ok(Some(_)));
println!("DEMONITOR stray={stray}");
println!("CLIENT DONE");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// The server's incarnation as this node sees it — for building pids by hand.
fn ev_incarnation() -> Incarnation {
smarm::cluster::membership::view()
.expect("manager up")
.into_iter()
.find(|i| i.name == "server")
.map(|i| i.incarnation)
.expect("server in view")
}
/// The Phase 4 c12 gate: remote monitors report the true reason, honour
/// corpses, leak nothing for unshipped pids, and cancel cleanly.
#[test]
fn remote_monitors_report_true_reasons() {
maybe_child(ROLES);
let mut server = spawn_node("server", &[]);
let saddr = server.wait_listening();
server.wait_line("READY", |l| l == "READY");
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
client.wait_line("DOWN exit Local(Exit)", |l| l == "DOWN exit Local(Exit)");
client.wait_line("DOWN panic Local(Panic)", |l| {
l == "DOWN panic Local(Panic)"
});
client.wait_line("DOWN corpse Local(Exit)", |l| {
l == "DOWN corpse Local(Exit)"
});
client.wait_line("DOWN hidden Local(NoProc)", |l| {
l == "DOWN hidden Local(NoProc)"
});
client.wait_line("DOWN bogus Local(NoProc)", |l| {
l == "DOWN bogus Local(NoProc)"
});
client.wait_line("DEMONITOR stray=false", |l| l == "DEMONITOR stray=false");
client.wait_line("CLIENT DONE", |l| l == "CLIENT DONE");
}
+254
View File
@@ -0,0 +1,254 @@
//! RFC 010 c15 — distributed pg: sync on `NodeUp`, incremental
//! `Join`/`Leave`, eager eviction announced, `NodeDown` sweep.
//!
//! Two nodes. The *origin* joins two local workers to `"pool"` before the
//! *observer* connects (so the observer's view comes from `Sync`), exposes a
//! `"go"` command inbox and then does exactly what the observer tells it:
//! kill one worker, join a third, leave with the second. The observer drives
//! that script through the cluster itself and asserts every step from
//! `members_all` — never touching the group on its own side, except once to
//! prove a mixed local+remote group reads correctly and that `members` stays
//! local. `dispatch_any` is exercised both ways: into the origin's worker
//! (remote pick, `send_to_remote`) and, once the origin is gone, into the
//! observer's own (local pick, `send_to`). Finally the parent SIGKILLs the
//! origin: the observer must sweep every remote member on `NodeDown`.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::expose::{expose, expose_type};
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::remote::{self, RemoteName};
use smarm::cluster::{
dispatch_any, members_all, pick_any, start, Config, DispatchAnyError, GroupMember, StaticSeeds,
Timing,
};
use smarm::{channel, join, leave, members, register, send_to, spawn_addr, Addressable, Name, Pid};
use std::time::{Duration, Instant};
const GO: Name<u8> = Name::new("go");
const POOL: &str = "pool";
/// A pool worker's message: `"die"` stops it, anything else is printed.
#[derive(Debug, PartialEq)]
struct Job(String);
struct Worker;
impl Addressable for Worker {
type Msg = Job;
}
impl serde::Serialize for Job {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
self.0.serialize(s)
}
}
impl<'de> serde::Deserialize<'de> for Job {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
String::deserialize(d).map(Job)
}
}
const ROLES: &[(&str, fn())] = &[("origin", role_origin), ("observer", role_observer)];
fn cfg(name: &str, seeds: Vec<(String, String)>) -> Config {
Config {
node_name: name.into(),
meta: NodeMeta {
role: "c15".into(),
region: "local".into(),
},
listen_addr: "127.0.0.1:0".into(),
strategy: Box::new(StaticSeeds::new(seeds)),
timing: Timing::default(),
}
}
/// A pool worker: prints every job it is handed, exits on `"die"`.
fn worker() -> Pid<Worker> {
spawn_addr::<Worker>(|rx| {
while let Ok(Job(s)) = rx.recv() {
if s == "die" {
return;
}
println!("JOB {s}");
}
})
}
fn role_origin() {
smarm::run(|| {
let cluster = start(cfg("origin", vec![])).expect("binds");
// Remote dispatch lands here only for a type this node accepts.
expose_type::<Job>();
let w1 = worker();
let w2 = worker();
assert!(join(POOL, w1));
assert!(join(POOL, w2));
let (go_tx, go_rx) = channel::<u8>();
register(GO, go_tx).unwrap();
expose(GO);
println!("LISTENING {}", cluster.local_addr());
println!("JOINED 2");
loop {
match go_rx.recv().unwrap() {
1 => {
send_to(w1, Job("die".into())).unwrap();
println!("KILLED w1");
}
2 => {
assert!(leave(POOL, w2));
println!("LEFT w2");
}
3 => {
let w3 = worker();
assert!(join(POOL, w3));
println!("JOINED w3");
}
n => panic!("unknown command {n}"),
}
}
});
}
fn remote_count(group: &str) -> usize {
members_all(group)
.iter()
.filter(|m| matches!(m, GroupMember::Remote(_)))
.count()
}
/// Cooperative poll until `pred`; panics (with the last view) on timeout.
fn wait_view(what: &str, group: &str, pred: impl Fn(&[GroupMember]) -> bool) {
let deadline = Instant::now() + Duration::from_secs(5);
loop {
let v = members_all(group);
if pred(&v) {
return;
}
assert!(
Instant::now() < deadline,
"timed out waiting for {what}; view = {v:?}"
);
smarm::sleep(Duration::from_millis(5));
}
}
fn role_observer() {
let origin_addr = std::env::var("SMARM_ORIGIN_ADDR").expect("SMARM_ORIGIN_ADDR");
smarm::run(move || {
let _cluster = start(cfg("observer", vec![("origin".into(), origin_addr)])).expect("binds");
let ev = subscribe().unwrap();
loop {
match ev.rx.recv().unwrap() {
NodeEvent::NodeUp(i) if i.name == "origin" => break,
_ => {}
}
}
let go = |n: u8| remote::send(RemoteName::new("origin", GO), n).unwrap();
// Sync: both pre-existing members arrive with no join on this side.
wait_view("sync of 2 remote members", POOL, |v| {
v.len() == 2 && v.iter().all(|m| matches!(m, GroupMember::Remote(_)))
});
let synced = members_all(POOL);
assert!(synced.iter().all(|m| match m {
GroupMember::Remote(p) => p.node() == "origin",
GroupMember::Local(_) => false,
}));
println!("SEES 2");
// Origin-side death: the origin's reaper announces the leave.
go(1);
wait_view("death evicted on observer", POOL, |v| v.len() == 1);
println!("SEES 1 after death");
// Incremental Join.
go(3);
wait_view("incremental join", POOL, |v| v.len() == 2);
println!("SEES 2 after join");
// Voluntary Leave.
go(2);
wait_view("incremental leave", POOL, |v| v.len() == 1);
println!("SEES 1 after leave");
// Mixed group: our own member sits beside the remote one in
// `members_all`; `members` stays local-only.
let me = worker();
assert!(join(POOL, me));
wait_view("mixed local+remote", POOL, |v| {
v.len() == 2 && v.contains(&GroupMember::Local(me.erase()))
});
assert_eq!(
members(POOL),
vec![me.erase()],
"local API never shows remotes"
);
assert_eq!(remote_count(POOL), 1);
println!("MIXED ok");
// dispatch_any: the store's first entry is the origin's w3 (it was
// announced before we joined), so the pick is remote and the job
// crosses the wire — the origin's worker prints it.
let picked = pick_any(POOL).expect("pool has members");
assert!(
matches!(picked, GroupMember::Remote(_)),
"first entry is remote: {picked:?}"
);
let reached = dispatch_any::<Worker>(POOL, Job("from-observer".into())).unwrap();
assert_eq!(reached, picked);
println!("DISPATCHED remote");
println!("PARK");
// Parent SIGKILLs the origin now: NodeDown must sweep its member,
// ours must survive.
wait_view("node_down sweep", POOL, |v| {
v == [GroupMember::Local(me.erase())]
});
assert_eq!(members(POOL), vec![me.erase()]);
println!("SWEPT");
// Now the only member is ours: a local pick, a local send.
let reached = dispatch_any::<Worker>(POOL, Job("local".into())).unwrap();
assert_eq!(reached, GroupMember::Local(me.erase()));
// And an empty group hands the message back.
match dispatch_any::<Worker>("nobody", Job("lost".into())) {
Err(DispatchAnyError::NoMember(Job(s))) => assert_eq!(s, "lost"),
other => panic!("expected NoMember, got {other:?}"),
}
println!("DISPATCHED local");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// The Phase 5 gate: sync, join, leave, death, node_down — all observed from
/// the peer, none of them a group operation on the peer — plus dispatch_any
/// reaching a remote member and a local one.
#[test]
fn groups_span_two_nodes() {
maybe_child(ROLES);
let mut origin = spawn_node("origin", &[]);
let addr = origin.wait_listening();
origin.wait_line("JOINED 2", |l| l == "JOINED 2");
let mut observer = spawn_node("observer", &[("SMARM_ORIGIN_ADDR", &addr)]);
observer.wait_line("SEES 2", |l| l == "SEES 2");
origin.wait_line("KILLED w1", |l| l == "KILLED w1");
observer.wait_line("SEES 1 after death", |l| l == "SEES 1 after death");
origin.wait_line("JOINED w3", |l| l == "JOINED w3");
observer.wait_line("SEES 2 after join", |l| l == "SEES 2 after join");
origin.wait_line("LEFT w2", |l| l == "LEFT w2");
observer.wait_line("SEES 1 after leave", |l| l == "SEES 1 after leave");
observer.wait_line("MIXED ok", |l| l == "MIXED ok");
observer.wait_line("DISPATCHED remote", |l| l == "DISPATCHED remote");
origin.wait_line("JOB from-observer", |l| l == "JOB from-observer");
observer.wait_line("PARK", |l| l == "PARK");
origin.kill();
observer.wait_line("SWEPT", |l| l == "SWEPT");
// Order between the root's line and the worker's is scheduling; wait
// for the later one to be certain both happened.
observer.wait_line("DISPATCHED local", |l| l == "DISPATCHED local");
observer.wait_line("JOB local", |l| l == "JOB local");
}
+356
View File
@@ -0,0 +1,356 @@
//! RFC 010 c10 — pid targeting + auto-serialization. The Phase 3 gate:
//! cross-node call/reply with no ceremony, under the subprocess harness.
//!
//! Local suite (`run()`, no network): serialize/deserialize shapes,
//! self-collapse, the outside-runtime contract, the local send-site
//! incarnation check with a probe proving **no frame is emitted**.
//!
//! Cross-process: two nodes. The *server* exposes a `Name<Req>`; the
//! *client* sends a `Req` carrying its own `Pid<Reply>` (auto-serialized to
//! a `RemotePid` on the wire); the server replies via `send_to_remote`
//! straight back to that pid — no name at the client end, no ceremony. A
//! third-node roundtrip: the client's pid travels client→server→relay→
//! server→client, and still delivers.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::{encode_payload, Frame, NodeMeta};
use smarm::cluster::expose::{expose, type_hash};
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::remote::{self, send_to_remote, RemoteName, RemotePid, ToRemoteError};
use smarm::cluster::{start, Config, StaticSeeds, Timing};
use smarm::pg::Incarnation;
use smarm::{channel, install, register, run, Addressable, Name, Pid};
use std::time::Duration;
// ---- message types (std-only payloads; the crate's serde is derive-less,
// so wire types are hand-rolled with serde's tuple/seq API via `serde::ser`
// impls below — the same thing a user's derive would generate) ------------
/// A request carrying a reply-to. Serialize/Deserialize are written by hand
/// here for exactly one reason: this crate deliberately does not pull in
/// serde-derive. Field 1 is the auto-serializing pid.
#[derive(Debug, PartialEq)]
struct Req {
text: String,
reply_to: RemotePid<Replier>,
}
#[derive(Debug, PartialEq)]
struct Reply(String);
struct Replier;
impl Addressable for Replier {
type Msg = Reply;
}
impl serde::Serialize for Req {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeTuple;
let mut t = s.serialize_tuple(2)?;
t.serialize_element(&self.text)?;
t.serialize_element(&self.reply_to)?;
t.end()
}
}
impl<'de> serde::Deserialize<'de> for Req {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let (text, reply_to) = <(String, RemotePid<Replier>)>::deserialize(d)?;
Ok(Req { text, reply_to })
}
}
impl serde::Serialize for Reply {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
self.0.serialize(s)
}
}
impl<'de> serde::Deserialize<'de> for Reply {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
String::deserialize(d).map(Reply)
}
}
// ================= local suite =========================================
/// A local `Pid<A>` serializes as a `RemotePid<A>` stamped with this node's
/// identity; deserializing it back on the same node collapses to the same
/// local pid (`local()` is `Some`, `Pid` round-trips).
#[test]
fn local_pid_serializes_and_collapses_on_self() {
maybe_child(ROLES);
run(|| {
// The local identity is set by cluster::start; the local suite sets
// it directly.
remote::set_local_identity("me", Incarnation::new(7));
let (tx, _rx) = channel::<Reply>();
let me: Pid<Replier> = install::<Replier>(tx);
let bytes = encode_payload(&me).unwrap();
let rp: RemotePid<Replier> = smarm::cluster::envelope::decode_payload(&bytes).unwrap();
assert_eq!(rp.node(), "me");
assert_eq!(rp.incarnation(), Incarnation::new(7));
assert_eq!(
rp.local(),
Some(me),
"self-node pid collapses to the local pid"
);
// Deserializing straight into Pid<A> works for a self-node pid...
let back: Pid<Replier> = smarm::cluster::envelope::decode_payload(&bytes).unwrap();
assert_eq!(back, me);
// ...and FAILS for a foreign one (collapse is literal: node == self).
let foreign = RemotePid::<Replier>::from_parts("elsewhere", Incarnation::new(1), 3, 1);
let fbytes = encode_payload(&foreign).unwrap();
assert!(smarm::cluster::envelope::decode_payload::<Pid<Replier>>(&fbytes).is_err());
assert_eq!(foreign.local(), None);
});
}
/// `send_to_remote` short-circuits locally for a self-node pid — the
/// zero-copy-equivalent collapse: the message object itself lands in the
/// local channel, no encode, no frame.
#[test]
fn send_to_remote_collapses_locally_for_self() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (tx, rx) = channel::<Reply>();
let me: Pid<Replier> = install::<Replier>(tx);
let rp = RemotePid::from_local(me).expect("identity set");
// Probe the outbound path: nothing must be handed to any connection.
let (probe_tx, probe_rx) = channel::<Frame>();
remote::bind_outbound_probe("me", Incarnation::new(7), probe_tx);
send_to_remote(rp, Reply("hi".into())).unwrap();
assert_eq!(rx.recv().unwrap(), Reply("hi".into()));
assert!(
matches!(probe_rx.try_recv(), Ok(None)),
"no frame for a local collapse"
);
});
}
/// RFC v2 §3: a `RemotePid` whose incarnation is not the current one for its
/// node fails at the local send site with `DeadIncarnation`, and NO frame
/// is emitted — asserted on a probe sender bound as that node's outbound.
#[test]
fn stale_incarnation_rejected_locally_no_frame() {
maybe_child(ROLES);
run(|| {
remote::set_local_identity("me", Incarnation::new(7));
let (probe_tx, probe_rx) = channel::<Frame>();
remote::bind_outbound_probe("peer", Incarnation::new(5), probe_tx);
let stale = RemotePid::<Replier>::from_parts("peer", Incarnation::new(4), 9, 1);
match send_to_remote(stale, Reply("late".into())) {
Err(ToRemoteError::DeadIncarnation(Reply(s))) => assert_eq!(s, "late"),
other => panic!("expected DeadIncarnation, got {other:?}"),
}
assert!(
matches!(probe_rx.try_recv(), Ok(None)),
"stale pid must emit no frame"
);
// The current incarnation goes through: a Send frame with the pid's
// (index, generation) and Reply's hash lands on the probe.
let live = RemotePid::<Replier>::from_parts("peer", Incarnation::new(5), 9, 1);
send_to_remote(live, Reply("now".into())).unwrap();
match probe_rx.recv().unwrap() {
Frame::Send {
index,
generation,
type_hash: h,
payload,
} => {
assert_eq!((index, generation), (9, 1));
assert_eq!(h, type_hash::<Reply>());
let r: Reply = smarm::cluster::envelope::decode_payload(&payload).unwrap();
assert_eq!(r, Reply("now".into()));
}
f => panic!("expected Send, got {f:?}"),
}
// Unknown node: NotConnected, no frame anywhere.
let nowhere = RemotePid::<Replier>::from_parts("nowhere", Incarnation::new(1), 1, 1);
assert!(matches!(
send_to_remote(nowhere, Reply("x".into())),
Err(ToRemoteError::NotConnected(_))
));
});
}
// ================= cross-process gate ==================================
const ROLES: &[(&str, fn())] = &[
("server", role_server),
("client", role_client),
("relay", role_relay),
];
const ECHO: Name<Req> = Name::new("c10.echo");
const RELAY: Name<Req> = Name::new("c10.relay");
fn cfg(name: &str, seeds: Vec<(String, String)>) -> Config {
Config {
node_name: name.to_string(),
meta: NodeMeta {
role: "c10".into(),
region: "local".into(),
},
listen_addr: std::env::var("SMARM_LISTEN_ADDR").unwrap_or_else(|_| "127.0.0.1:0".into()),
strategy: Box::new(StaticSeeds::new(seeds)),
timing: Timing::default(),
}
}
fn wait_up(events: &smarm::cluster::membership::MembershipEvents, who: &str) {
loop {
match events.rx.recv() {
Ok(NodeEvent::NodeUp(i)) if i.name == who => return,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
}
/// Server: exposes ECHO; each Req is answered by `send_to_remote` to its
/// reply_to — the server never learns a name for the client. If the Req text
/// starts with "via-relay:", it forwards the whole Req (reply_to and all) to
/// the relay node instead, which sends it back here; the second arrival is
/// answered normally. That is the pid's third-node roundtrip.
fn role_server() {
let relay_addr = std::env::var("SMARM_RELAY_ADDR").ok();
smarm::run(move || {
let seeds = relay_addr
.map(|a| vec![("relay".to_string(), a)])
.unwrap_or_default();
let cluster = start(cfg("server", seeds)).expect("binds");
println!("LISTENING {}", cluster.local_addr());
let (tx, rx) = channel::<Req>();
register(ECHO, tx).unwrap();
expose(ECHO);
println!("READY");
loop {
let req = rx.recv().unwrap();
if let Some(rest) = req.text.strip_prefix("via-relay:") {
let fwd = Req {
text: format!("relayed:{rest}"),
reply_to: req.reply_to,
};
remote::send(RemoteName::new("relay", RELAY), fwd).unwrap();
println!("FORWARDED");
continue;
}
println!("REQ {}", req.text);
send_to_remote(req.reply_to, Reply(format!("echo:{}", req.text))).unwrap();
}
});
}
/// Relay: exposes RELAY; bounces every Req straight back to the server's
/// ECHO, untouched. The client's pid inside it now crosses relay→server.
fn role_relay() {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
smarm::run(move || {
let cluster = start(cfg("relay", vec![("server".into(), server_addr)])).expect("binds");
println!("LISTENING {}", cluster.local_addr());
let (tx, rx) = channel::<Req>();
register(RELAY, tx).unwrap();
expose(RELAY);
let ev = subscribe().unwrap();
wait_up(&ev, "server");
println!("READY");
loop {
let req = rx.recv().unwrap();
println!("RELAYING {}", req.text);
remote::send(RemoteName::new("server", ECHO), req).unwrap();
}
});
}
/// Client: connects to server, installs a Reply inbox on its own pid,
/// declares it accepts `Reply` (`expose_type` — the RFC's one kept piece of
/// ceremony: nothing is remotely deliverable by default), sends a Req with
/// `reply_to = my pid` (auto-serialized), awaits the reply.
fn role_client() {
let server_addr = std::env::var("SMARM_SERVER_ADDR").expect("SMARM_SERVER_ADDR");
let via_relay = std::env::var("SMARM_VIA_RELAY").is_ok();
smarm::run(move || {
let _cluster = start(cfg("client", vec![("server".into(), server_addr)])).expect("binds");
let ev = subscribe().unwrap();
wait_up(&ev, "server");
println!("MEMBER-UP server");
let (tx, rx) = channel::<Reply>();
let me: Pid<Replier> = install::<Replier>(tx);
// The one deliberate line: a pid-targeted inbound is deliverable only
// for types this node has said it accepts (RFC §4, the safety).
smarm::cluster::expose::expose_type::<Reply>();
let text = if via_relay { "via-relay:ping" } else { "ping" };
remote::send(
RemoteName::new("server", ECHO),
Req {
text: text.into(),
reply_to: RemotePid::from_local(me).expect("identity set"),
},
)
.unwrap();
println!("SENT");
let Reply(s) = rx.recv().unwrap();
println!("REPLY {s}");
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// The gate: cross-node call/reply with no ceremony.
#[test]
fn cross_node_call_reply_no_ceremony() {
maybe_child(ROLES);
let mut server = spawn_node("server", &[]);
let saddr = server.wait_listening();
server.wait_line("READY", |l| l == "READY");
let mut client = spawn_node("client", &[("SMARM_SERVER_ADDR", &saddr)]);
client.wait_line("SENT", |l| l == "SENT");
server.wait_line("REQ ping", |l| l == "REQ ping");
client.wait_line("REPLY echo:ping", |l| l == "REPLY echo:ping");
}
/// The client's pid, round-tripped through a third node, still delivers.
#[test]
fn pid_roundtrips_through_third_node() {
maybe_child(ROLES);
// Relay needs the server address; server needs the relay address —
// pre-reserve the relay port (same accepted micro-window as cluster_mesh).
let relay_addr = {
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
l.local_addr().unwrap().to_string()
};
let mut server = spawn_node("server", &[("SMARM_RELAY_ADDR", &relay_addr)]);
let saddr = server.wait_listening();
server.wait_line("READY", |l| l == "READY");
let mut relay = spawn_node(
"relay",
&[
("SMARM_SERVER_ADDR", &saddr),
("SMARM_LISTEN_ADDR", &relay_addr),
],
);
let _ = relay.wait_listening();
relay.wait_line("READY", |l| l == "READY");
let mut client = spawn_node(
"client",
&[("SMARM_SERVER_ADDR", &saddr), ("SMARM_VIA_RELAY", "1")],
);
client.wait_line("SENT", |l| l == "SENT");
server.wait_line("FORWARDED", |l| l == "FORWARDED");
relay.wait_line("RELAYING", |l| l.starts_with("RELAYING"));
server.wait_line("REQ relayed:ping", |l| l == "REQ relayed:ping");
client.wait_line("REPLY echo:relayed:ping", |l| {
l == "REPLY echo:relayed:ping"
});
}
+221
View File
@@ -0,0 +1,221 @@
//! RFC 010 c9 — remote `Name` sends: the outbound seam and the single
//! inbound name-resolution seam, cross-process.
//!
//! Two node processes each run the integrated `cluster::start`. The
//! *receiver* registers a `String` inbox under a name and exposes it (and
//! registers a second name it does NOT expose); the *sender* waits for
//! `node_up`, then sends. Facts cross as stdout lines: `LISTENING <addr>`,
//! `MEMBER-UP <name>`, `GOT <payload>`, `SEND-RESULT <case> <verdict>`.
//! Roles park forever afterwards (retractable-state trap); the parent
//! SIGKILLs via `Drop`.
//!
//! What is asserted at each end (roadmap-binding):
//! - cross-node name-send delivers the payload;
//! - an unexposed name is unreachable — the receiver's inbox stays empty
//! even though the name IS registered locally;
//! - a wrong type hash is a decode failure at the receiver, never a
//! misroute — the `String` inbox does not see a `u64` delivered under a
//! made-up hash, nor a `u64` under `u64`'s hash;
//! - a send to a disconnected (never-connected) node fails locally with
//! `NotConnected`, and `Ok(())` means only "handed to the transport".
//!
//! Timing note for the "stays empty" assertions: they are proven by
//! ORDERING, not by waiting — the sender emits the negative-case frames
//! BEFORE the positive one on the same connection (in-order stream), so when
//! the receiver has seen the positive payload, the negatives have already
//! been processed and refused. No sleep-and-hope.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node, Node};
use smarm::cluster::envelope::NodeMeta;
use smarm::cluster::expose::expose;
use smarm::cluster::membership::{subscribe, NodeEvent};
use smarm::cluster::remote::{send_remote_raw, RemoteName, RemoteSendError};
use smarm::cluster::{start, Config, StaticSeeds, Timing};
use smarm::{channel, register, Name};
use std::time::Duration;
const ROLES: &[(&str, fn())] = &[("receiver", role_receiver), ("sender", role_sender)];
const INBOX: Name<String> = Name::new("c9.inbox");
const HIDDEN: Name<String> = Name::new("c9.hidden");
fn base_config(name: &str, seeds: Vec<(String, String)>) -> Config {
Config {
node_name: name.to_string(),
meta: NodeMeta {
role: "c9".to_string(),
region: "local".to_string(),
},
listen_addr: "127.0.0.1:0".to_string(),
strategy: Box::new(StaticSeeds::new(seeds)),
timing: Timing::default(),
}
}
/// Receiver: register + expose INBOX; register HIDDEN unexposed **in a
/// separate actor** (one actor holds one channel per message type — a
/// second `register` of the same `M` on one actor silently replaces the
/// first, closing it); print every payload that lands in either.
fn role_receiver() {
smarm::run(|| {
let cluster = start(base_config("recv", vec![])).expect("listener binds");
println!("LISTENING {}", cluster.local_addr());
// HIDDEN's holder: its own actor, so its String channel does not
// displace INBOX's on the root actor.
let (hidden_ready_tx, hidden_ready_rx) = channel::<()>();
smarm::spawn(move || {
let (hid_tx, hid_rx) = channel::<String>();
register(HIDDEN, hid_tx).unwrap();
hidden_ready_tx.send(()).unwrap();
loop {
match hid_rx.recv() {
Ok(s) => println!("GOT-HIDDEN {s}"),
Err(_) => break,
}
}
});
hidden_ready_rx.recv().unwrap();
let (in_tx, in_rx) = channel::<String>();
register(INBOX, in_tx).unwrap();
expose(INBOX);
println!("READY");
loop {
match in_rx.recv() {
Ok(s) => println!("GOT {s}"),
Err(_) => break,
}
}
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
/// Sender: connect to recv, wait for node_up, then in this ORDER on the one
/// connection: hidden-name send, wrong-hash sends (two flavours), then the
/// positive send. Plus a send to a node that is not connected at all.
fn role_sender() {
let recv_addr = std::env::var("SMARM_RECV_ADDR").expect("SMARM_RECV_ADDR");
smarm::run(move || {
let _cluster = start(base_config("send", vec![("recv".to_string(), recv_addr)]))
.expect("listener binds");
let events = subscribe().expect("manager is up");
loop {
match events.rx.recv() {
Ok(NodeEvent::NodeUp(info)) if info.name == "recv" => break,
Ok(_) => continue,
Err(_) => panic!("manager gone"),
}
}
println!("MEMBER-UP recv");
// Not connected: purely local knowledge, no frame leaves.
let ghost: RemoteName<String> = RemoteName::new("nowhere", INBOX);
let r = smarm::cluster::remote::send(ghost, "lost".to_string());
println!(
"SEND-RESULT not-connected {}",
match r {
Err(RemoteSendError::NotConnected(_)) => "NotConnected",
Ok(()) => "Ok",
Err(_) => "OtherErr",
}
);
// Unexposed name at the peer: the frame goes (local knowledge can't
// know the peer's exposed set) and the peer refuses it.
let hidden: RemoteName<String> = RemoteName::new("recv", HIDDEN);
let r = smarm::cluster::remote::send(hidden, "should not land".to_string());
println!(
"SEND-RESULT hidden {}",
if r.is_ok() { "Ok" } else { "Err" }
);
// Wrong hash, two flavours: (a) a u64 payload under a made-up hash
// (unknown type at the peer); (b) a u64 payload under u64's real
// hash against a String-typed name (decoder known, wrong channel).
// Both are raw sends — the typed API cannot express them, by design.
let bogus = 0xdead_beef_u64;
let r = send_remote_raw(
"recv",
"c9.inbox",
bogus,
&smarm::cluster::envelope::encode_payload(&7u64).unwrap(),
);
println!(
"SEND-RESULT wrong-hash-unknown {}",
if r.is_ok() { "Ok" } else { "Err" }
);
let r = send_remote_raw(
"recv",
"c9.inbox",
smarm::cluster::expose::type_hash::<u64>(),
&smarm::cluster::envelope::encode_payload(&7u64).unwrap(),
);
println!(
"SEND-RESULT wrong-hash-known {}",
if r.is_ok() { "Ok" } else { "Err" }
);
// Positive: last on the stream, so its arrival proves the negatives
// were already processed.
let inbox: RemoteName<String> = RemoteName::new("recv", INBOX);
let r = smarm::cluster::remote::send(inbox, "hello from send".to_string());
println!(
"SEND-RESULT positive {}",
if r.is_ok() { "Ok" } else { "Err" }
);
loop {
smarm::sleep(Duration::from_secs(3600));
}
});
}
fn wait_send_result(node: &mut Node, case: &str) -> String {
let prefix = format!("SEND-RESULT {case} ");
let line = node.wait_line(&prefix, |l| l.starts_with(&prefix));
line[prefix.len()..].to_string()
}
#[test]
fn remote_name_send_delivers_and_refusals_never_misroute() {
maybe_child(ROLES);
let mut recv = spawn_node("receiver", &[]);
let addr = recv.wait_listening();
recv.wait_line("READY", |l| l == "READY");
let mut send = spawn_node("sender", &[("SMARM_RECV_ADDR", &addr)]);
send.wait_line("MEMBER-UP recv", |l| l == "MEMBER-UP recv");
// Local-knowledge-only failure for an unknown node.
assert_eq!(wait_send_result(&mut send, "not-connected"), "NotConnected");
// Every frame-bearing send is Ok — Ok means "handed to the transport",
// nothing about what the peer does with it (RFC §3, documented here).
assert_eq!(wait_send_result(&mut send, "hidden"), "Ok");
assert_eq!(wait_send_result(&mut send, "wrong-hash-unknown"), "Ok");
assert_eq!(wait_send_result(&mut send, "wrong-hash-known"), "Ok");
assert_eq!(wait_send_result(&mut send, "positive"), "Ok");
// The positive payload lands...
recv.wait_line("GOT hello from send", |l| l == "GOT hello from send");
// ...and, by stream ordering, every negative before it was refused: no
// GOT for the wrong-hash frames, no GOT-HIDDEN at all. The transcript
// up to this point is the proof.
let transcript = recv.transcript();
let gots: Vec<&str> = transcript
.iter()
.map(|s| s.as_str())
.filter(|l| l.starts_with("GOT"))
.collect();
assert_eq!(
gots,
["GOT hello from send"],
"exactly one delivery, the exposed one"
);
}
+272
View File
@@ -0,0 +1,272 @@
//! RFC 010 c3 — transport conformance suite, run against both shipped impls
//! (TCP and in-memory loopback), plus impl-specific cases.
//!
//! Shared suite (roadmap): frame roundtrips through the framed codec, framing
//! across a split write, coalesced frames in one write, peer-close mid-frame
//! (must error, not EOF), clean close at a frame boundary (EOF as `Ok(None)`).
//!
//! The TCP impl parks the calling actor, so its runs live inside `smarm::run`;
//! loopback blocks the OS thread and runs as plain tests.
#![cfg(feature = "cluster")]
use smarm::cluster::envelope::Frame;
use smarm::cluster::transport::loopback::LoopbackTransport;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{Conn, FramedConn, RecvError, Transport};
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/// Listener + dial + accept against one transport, both conns returned.
/// Relies on dial not requiring a concurrent accept (TCP backlog / loopback
/// queue), so a single thread or actor can hold both ends.
fn pair(t: &dyn Transport, addr: &str) -> (Box<dyn Conn>, Box<dyn Conn>) {
let mut l = t.listen(addr).unwrap();
let a = t.dial(&l.local_addr()).unwrap();
let b = l.accept().unwrap();
(a, b)
}
fn frames() -> Vec<Frame> {
vec![
Frame::Heartbeat,
Frame::Send {
index: 42,
generation: 3,
type_hash: 0x1234_5678_9ABC_DEF0,
payload: vec![1, 2, 3, 4, 5],
},
Frame::SendNamed {
name: "the_counter".into(),
type_hash: 0xFFFF_0000_FFFF_0000,
payload: vec![],
},
Frame::Demonitor { monitor_id: 77 },
]
}
fn encode(f: &Frame) -> Vec<u8> {
let mut out = Vec::new();
f.encode(&mut out).unwrap();
out
}
// ---------------------------------------------------------------------------
// Shared conformance suite — generic over an established pair
// ---------------------------------------------------------------------------
fn suite_roundtrip(a: Box<dyn Conn>, b: Box<dyn Conn>) {
let mut fa = FramedConn::new(a);
let mut fb = FramedConn::new(b);
// a -> b, then b -> a: both directions carry every frame shape.
for f in frames() {
fa.send(&f).unwrap();
assert_eq!(fb.recv().unwrap().unwrap(), f);
}
for f in frames() {
fb.send(&f).unwrap();
assert_eq!(fa.recv().unwrap().unwrap(), f);
}
}
fn suite_split_write(mut a: Box<dyn Conn>, b: Box<dyn Conn>) {
let f = Frame::Send {
index: 7,
generation: 1,
type_hash: 0xAB,
payload: vec![9; 64],
};
let bytes = encode(&f);
// Split inside the length prefix, then inside the body: the reader must
// reassemble regardless of where the boundary falls.
a.write_all(&bytes[..2]).unwrap();
a.write_all(&bytes[2..10]).unwrap();
a.write_all(&bytes[10..]).unwrap();
let mut fb = FramedConn::new(b);
assert_eq!(fb.recv().unwrap().unwrap(), f);
}
fn suite_coalesced(mut a: Box<dyn Conn>, b: Box<dyn Conn>) {
let f1 = Frame::Heartbeat;
let f2 = Frame::Demonitor { monitor_id: 5 };
let mut bytes = encode(&f1);
bytes.extend_from_slice(&encode(&f2));
a.write_all(&bytes).unwrap();
let mut fb = FramedConn::new(b);
assert_eq!(fb.recv().unwrap().unwrap(), f1);
assert_eq!(fb.recv().unwrap().unwrap(), f2);
}
fn suite_close_mid_frame(mut a: Box<dyn Conn>, b: Box<dyn Conn>) {
let bytes = encode(&Frame::Send {
index: 1,
generation: 1,
type_hash: 1,
payload: vec![0; 128],
});
a.write_all(&bytes[..bytes.len() / 2]).unwrap();
a.close();
let mut fb = FramedConn::new(b);
match fb.recv() {
Err(RecvError::TruncatedByPeer) => {}
other => panic!("expected TruncatedByPeer, got {other:?}"),
}
}
fn suite_clean_close(mut a: Box<dyn Conn>, b: Box<dyn Conn>) {
let f = Frame::Heartbeat;
a.write_all(&encode(&f)).unwrap();
a.close();
let mut fb = FramedConn::new(b);
// The buffered frame is still delivered, then EOF at the boundary.
assert_eq!(fb.recv().unwrap().unwrap(), f);
assert!(fb.recv().unwrap().is_none());
}
fn run_suite(t: &dyn Transport, addr: &str) {
let (a, b) = pair(t, addr);
suite_roundtrip(a, b);
let (a, b) = pair(t, addr);
suite_split_write(a, b);
let (a, b) = pair(t, addr);
suite_coalesced(a, b);
let (a, b) = pair(t, addr);
suite_close_mid_frame(a, b);
let (a, b) = pair(t, addr);
suite_clean_close(a, b);
}
// ---------------------------------------------------------------------------
// Loopback — plain tests, no runtime
// ---------------------------------------------------------------------------
#[test]
fn loopback_conformance() {
// Fresh transport per pair() call is fine, but one instance must also
// support sequential re-listen on distinct addresses.
let t = LoopbackTransport::default();
run_suite(&t, "alpha");
}
#[test]
fn loopback_dial_unknown_addr_refused() {
let t = LoopbackTransport::default();
let err = t.dial("nobody-home").unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::ConnectionRefused);
}
#[test]
fn loopback_addr_in_use() {
let t = LoopbackTransport::default();
let _l = t.listen("alpha").unwrap();
let err = t.listen("alpha").unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
}
#[test]
fn loopback_listener_drop_frees_addr_and_refuses_dial() {
let t = LoopbackTransport::default();
let l = t.listen("alpha").unwrap();
drop(l);
let err = t.dial("alpha").unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::ConnectionRefused);
// Address is reusable after the listener is gone.
let _l2 = t.listen("alpha").unwrap();
}
#[test]
fn loopback_write_after_peer_close_broken_pipe() {
let t = LoopbackTransport::default();
let (mut a, mut b) = pair(&t, "alpha");
b.close();
let err = a.write_all(&[1, 2, 3]).unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe);
}
#[test]
fn loopback_cross_thread_blocking_read() {
// Reader blocks on an empty pipe until the writer thread delivers.
let t = LoopbackTransport::default();
let (a, b) = pair(&t, "alpha");
let mut fb = FramedConn::new(b);
let writer = std::thread::spawn(move || {
let mut a = a;
std::thread::sleep(std::time::Duration::from_millis(30));
a.write_all(&encode(&Frame::Heartbeat)).unwrap();
});
assert_eq!(fb.recv().unwrap().unwrap(), Frame::Heartbeat);
writer.join().unwrap();
}
// ---------------------------------------------------------------------------
// TCP — inside the runtime (read/write park the calling actor)
// ---------------------------------------------------------------------------
#[test]
fn tcp_conformance() {
smarm::run(|| {
run_suite(&TcpTransport, "127.0.0.1:0");
});
}
#[test]
fn tcp_dial_refused() {
smarm::run(|| {
// Bind to an OS-assigned port, learn it, close the listener, dial it.
let addr = {
let l = TcpTransport.listen("127.0.0.1:0").unwrap();
l.local_addr()
};
let err = TcpTransport.dial(&addr).unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::ConnectionRefused);
});
}
#[test]
fn tcp_bad_addr_rejected_without_resolution() {
// Addresses are opaque pre-resolved strings; the c9 seam resolves names.
// A hostname is therefore invalid input here, not something to resolve.
let err = TcpTransport.dial("localhost:1234").unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
}
#[test]
fn tcp_local_addr_reports_real_port() {
let l = TcpTransport.listen("127.0.0.1:0").unwrap();
let addr = l.local_addr();
let port: u16 = addr.rsplit(':').next().unwrap().parse().unwrap();
assert_ne!(port, 0);
}
#[test]
fn tcp_big_frame_across_socket_buffers() {
// A payload far beyond socket buffer sizes forces genuine fragmentation
// and write backpressure: writer and reader must run concurrently.
smarm::run(|| {
let (tx, rx) = smarm::channel::<Frame>();
let payload = vec![0xA5u8; 4 * 1024 * 1024];
let f = Frame::Send {
index: 9,
generation: 2,
type_hash: 0xC0FFEE,
payload,
};
let mut l = TcpTransport.listen("127.0.0.1:0").unwrap();
let addr = l.local_addr();
let fw = f.clone();
let writer = smarm::spawn(move || {
let mut fa = FramedConn::new(TcpTransport.dial(&addr).unwrap());
fa.send(&fw).unwrap();
});
let reader = smarm::spawn(move || {
let mut fb = FramedConn::new(l.accept().unwrap());
let got = fb.recv().unwrap().unwrap();
tx.send(got).unwrap();
});
let got = rx.recv().unwrap();
assert_eq!(got, f);
writer.join().unwrap();
reader.join().unwrap();
});
}
+108
View File
@@ -0,0 +1,108 @@
//! RFC 010 c4 — two-node harness smoke tests.
//!
//! Roadmap: "spawn two, handshake-less connect, both exit clean." The
//! listener node binds port 0 and announces its concrete address; the
//! dialer connects raw (no Hello — c5 doesn't exist yet), pushes one
//! Heartbeat through the real framed codec, and closes. Assertions are on
//! protocol-visible lines only. Flake budget: see tests/common/mod.rs.
#![cfg(feature = "cluster")]
mod common;
use common::{maybe_child, spawn_node};
use smarm::cluster::envelope::Frame;
use smarm::cluster::transport::tcp::TcpTransport;
use smarm::cluster::transport::{FramedConn, Transport};
const ROLES: &[(&str, fn())] = &[
("listener", role_listener),
("dialer", role_dialer),
("hang", role_hang),
("fail", role_fail),
];
fn role_listener() {
smarm::run(|| {
let mut l = TcpTransport.listen("127.0.0.1:0").unwrap();
println!("LISTENING {}", l.local_addr());
let mut fc = FramedConn::new(l.accept().unwrap());
match fc.recv() {
Ok(Some(Frame::Heartbeat)) => println!("RECV heartbeat"),
other => {
println!("RECV unexpected: {other:?}");
std::process::exit(3);
}
}
match fc.recv() {
Ok(None) => println!("PEER-CLOSED clean"),
other => {
println!("PEER-CLOSED unexpected: {other:?}");
std::process::exit(3);
}
}
});
println!("EXIT ok");
}
fn role_dialer() {
let addr = std::env::var("SMARM_PEER_ADDR").expect("SMARM_PEER_ADDR not set");
smarm::run(move || {
let mut fc = FramedConn::new(TcpTransport.dial(&addr).unwrap());
fc.send(&Frame::Heartbeat).unwrap();
fc.close();
println!("SENT heartbeat");
});
println!("EXIT ok");
}
fn role_hang() {
println!("HANGING");
loop {
std::thread::sleep(std::time::Duration::from_secs(3600));
}
}
fn role_fail() {
std::process::exit(7);
}
/// The roadmap smoke test: two real processes, raw transport connect, one
/// frame across, clean close observed on both sides, both exit 0.
#[test]
fn two_nodes_connect_and_exit_clean() {
maybe_child(ROLES);
let mut listener = spawn_node("listener", &[]);
let addr = listener.wait_listening();
let mut dialer = spawn_node("dialer", &[("SMARM_PEER_ADDR", &addr)]);
dialer.wait_line("SENT heartbeat", |l| l == "SENT heartbeat");
listener.wait_line("RECV heartbeat", |l| l == "RECV heartbeat");
listener.wait_line("clean peer close", |l| l == "PEER-CLOSED clean");
dialer.wait_exit_ok();
listener.wait_exit_ok();
}
/// Reap guarantee: dropping a Node kills a hung child — no orphan survives
/// a panicking test.
#[test]
fn drop_reaps_hung_node() {
maybe_child(ROLES);
let mut node = spawn_node("hang", &[]);
node.wait_line("HANGING", |l| l == "HANGING");
let pid = node.pid().expect("live child has a pid") as libc::pid_t;
drop(node);
// After Drop's kill+wait the pid is fully reaped: signalling it fails
// with ESRCH (pid-reuse in this instant is not a realistic race).
let rc = unsafe { libc::kill(pid, 0) };
assert_eq!(rc, -1, "process still signallable after Drop");
let errno = std::io::Error::last_os_error().raw_os_error();
assert_eq!(errno, Some(libc::ESRCH), "expected ESRCH, got {errno:?}");
}
/// Nonzero child exits surface as statuses, not hangs or panics.
#[test]
fn nonzero_exit_is_reported() {
maybe_child(ROLES);
let mut node = spawn_node("fail", &[]);
let status = node.wait_exit();
assert_eq!(status.code(), Some(7));
}
+250
View File
@@ -0,0 +1,250 @@
//! RFC 010 c4 — subprocess multi-node test harness.
//!
//! The runtime is a process singleton, so two real nodes means two
//! processes. This harness re-execs the *current test binary* as node
//! processes (precedent: tests/stack_diag.rs), tails their output live,
//! waits on protocol-visible lines, and reaps reliably no matter how the
//! test dies.
//!
//! Usage, per test file:
//!
//! - Declare roles as plain `fn()`s. A role prints protocol-visible facts
//! as single lines (Rust's piped stdout is line-buffered, so `println!`
//! is enough) and exits.
//! - **Every** `#[test]` in the file starts with
//! [`maybe_child`]`(ROLES)` — in the child re-exec, whichever test
//! libtest runs first performs the role and exits before the rest of the
//! suite runs (children are spawned with `--test-threads=1 --quiet`).
//! - The parent side spawns nodes with [`spawn_node`], waits on lines with
//! [`Node::wait_line`], and on exits with [`Node::wait_exit`].
//!
//! Port assignment: children bind port 0 and *report* the concrete address
//! (e.g. `LISTENING 127.0.0.1:41733`) rather than the parent pre-picking a
//! port — no bind/steal race by construction.
//!
//! Reaping: [`Node`]'s `Drop` SIGKILLs and `wait(2)`s the child, so a
//! panicking test (including a `wait_line` timeout) leaves no orphan and
//! no zombie. Tail threads exit on pipe EOF.
//!
//! Flake budget (explicit, per roadmap): every wait is bounded by
//! [`WAIT`] (10 s) against a typical cost of well under 1 s; the smoke
//! suite ran 10/10 clean at authoring time. Treat >1 failure in 100 runs
//! as a harness or runtime regression, not weather. On timeout the panic
//! message carries the node's full transcript so far.
#![allow(dead_code)] // Reusable surface: later phases use more of it than any one file.
use std::env;
use std::io::{BufRead, BufReader};
use std::process::{Child, Command, ExitStatus, Stdio};
use std::sync::mpsc::{Receiver, RecvTimeoutError};
use std::time::{Duration, Instant};
/// Env var selecting the child role in a re-exec.
const ROLE_ENV: &str = "SMARM_TWO_NODE_ROLE";
/// Upper bound for every wait in the harness. See the flake budget above.
pub const WAIT: Duration = Duration::from_secs(10);
/// In the child re-exec: run the matching role and exit. In the parent (no
/// role env set): return immediately. Call this first in every `#[test]` of
/// any file using the harness, passing the file's full role table.
pub fn maybe_child(roles: &[(&str, fn())]) {
let role = match env::var(ROLE_ENV) {
Ok(r) => r,
Err(_) => return,
};
for (name, f) in roles {
if *name == role {
f();
std::process::exit(0);
}
}
eprintln!("two_node harness: unknown role {role:?}");
std::process::exit(2);
}
/// One spawned node process with live-tailed output.
pub struct Node {
/// Role name, for panic messages.
pub role: String,
child: Option<Child>,
stdout_rx: Receiver<String>,
stderr_rx: Receiver<String>,
/// Every line consumed from stdout/stderr so far, for failure dumps.
transcript: Vec<String>,
}
fn tail(stream: impl std::io::Read + Send + 'static, prefix: &'static str) -> Receiver<String> {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for line in BufReader::new(stream).lines() {
let line = match line {
Ok(l) => l,
Err(_) => break,
};
// Receiver gone (Node dropped): stop tailing.
if tx.send(format!("{prefix}{line}")).is_err() {
break;
}
}
});
rx
}
/// Re-exec the current test binary as `role`, with any extra env vars.
pub fn spawn_node(role: &str, extra_env: &[(&str, &str)]) -> Node {
let exe = env::current_exe().expect("current_exe");
let mut cmd = Command::new(exe);
cmd.env(ROLE_ENV, role)
// --test-threads=1: exactly one test fn starts, hits maybe_child,
// and becomes the role. --nocapture: libtest must not swallow the
// role's println! lines — the parent tails them live.
.args(["--test-threads=1", "--quiet", "--nocapture"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
for (k, v) in extra_env {
cmd.env(k, v);
}
let mut child = cmd.spawn().expect("failed to spawn node process");
let stdout_rx = tail(child.stdout.take().expect("piped stdout"), "");
let stderr_rx = tail(child.stderr.take().expect("piped stderr"), "[stderr] ");
Node {
role: role.to_string(),
child: Some(child),
stdout_rx,
stderr_rx,
transcript: Vec::new(),
}
}
impl Node {
fn drain_stderr(&mut self) {
while let Ok(l) = self.stderr_rx.try_recv() {
self.transcript.push(l);
}
}
fn dump(&self) -> String {
if self.transcript.is_empty() {
"<no output>".to_string()
} else {
self.transcript.join("\n")
}
}
/// Every stdout/stderr line seen so far, in arrival order. For
/// ordering-proof assertions ("by the time X arrived, Y had not").
#[allow(dead_code)]
pub fn transcript(&self) -> &[String] {
&self.transcript
}
/// Wait until a stdout line satisfies `pred`; return it. Panics with the
/// full transcript after [`WAIT`]. `what` names the expectation in the
/// panic message.
pub fn wait_line(&mut self, what: &str, pred: impl Fn(&str) -> bool) -> String {
let deadline = Instant::now() + WAIT;
loop {
self.drain_stderr();
let left = deadline.saturating_duration_since(Instant::now());
match self.stdout_rx.recv_timeout(left) {
Ok(line) => {
self.transcript.push(line.clone());
if pred(&line) {
return line;
}
}
Err(RecvTimeoutError::Timeout) => {
// Pull in whatever stderr arrived since the last drain,
// so a role's eprintln! diagnostics survive into the dump.
self.drain_stderr();
panic!(
"node {:?}: timed out waiting for {what} after {WAIT:?}; transcript:\n{}",
self.role,
self.dump()
);
}
Err(RecvTimeoutError::Disconnected) => {
self.drain_stderr();
panic!(
"node {:?}: output closed while waiting for {what}; transcript:\n{}",
self.role,
self.dump()
);
}
}
}
}
/// Shorthand: wait for a `LISTENING <addr>` announcement, return the addr.
pub fn wait_listening(&mut self) -> String {
let line = self.wait_line("LISTENING announcement", |l| l.starts_with("LISTENING "));
line["LISTENING ".len()..].to_string()
}
/// Wait for the process to exit; panics with the transcript on timeout.
pub fn wait_exit(&mut self) -> ExitStatus {
let deadline = Instant::now() + WAIT;
loop {
let polled = match self.child.as_mut() {
Some(c) => c.try_wait(),
None => panic!("node {:?}: already reaped", self.role),
};
match polled {
Ok(Some(status)) => {
// Drain remaining output into the transcript for dumps.
self.drain_stderr();
while let Ok(l) = self.stdout_rx.try_recv() {
self.transcript.push(l);
}
self.child = None;
return status;
}
Ok(None) => {
if Instant::now() >= deadline {
self.drain_stderr();
self.kill();
panic!(
"node {:?}: did not exit within {WAIT:?}; transcript:\n{}",
self.role,
self.dump()
);
}
std::thread::sleep(Duration::from_millis(10));
}
Err(e) => panic!("node {:?}: try_wait failed: {e}", self.role),
}
}
}
/// Wait for exit and require success, dumping the transcript otherwise.
pub fn wait_exit_ok(&mut self) {
let status = self.wait_exit();
assert!(
status.success(),
"node {:?}: exited with {status}; transcript:\n{}",
self.role,
self.dump()
);
}
/// The child's OS pid, if not yet reaped.
pub fn pid(&self) -> Option<u32> {
self.child.as_ref().map(Child::id)
}
/// SIGKILL + reap now (idempotent).
pub fn kill(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
impl Drop for Node {
fn drop(&mut self) {
self.kill();
}
}
+13 -20
View File
@@ -1,5 +1,6 @@
//! Process-group tests that run under the scheduler: `join` installs a real //! Process-group tests that run under the scheduler: `join` installs a real
//! monitor on a live actor, and a real death drives eviction on next contact. //! monitor on a live actor, and a real death drives eviction (the reaper
//! actor sweeps it; the read path hides it in the meantime).
//! (Pure structural invariants live in the `pg` unit tests.) //! (Pure structural invariants live in the `pg` unit tests.)
use smarm::{channel, members, pick, run, spawn}; use smarm::{channel, members, pick, run, spawn};
@@ -35,19 +36,15 @@ fn a_dead_actor_vanishes_from_every_group_it_joined() {
assert_eq!(members("g1"), vec![pid]); assert_eq!(members("g1"), vec![pid]);
assert_eq!(members("g2"), vec![pid]); assert_eq!(members("g2"), vec![pid]);
// Release and reap the actor. finalize_actor queues the Down to our // Release the actor. finalize_actor queues the Down to the reaper and
// monitors before unparking joiners, so by the time join() returns the // marks the slot dead before unparking joiners, so by the time join()
// Down is already waiting in the membership channel. // returns every read hides the pid whether or not the reaper has run.
tx.send(()).unwrap(); tx.send(()).unwrap();
h.join().unwrap(); h.join().unwrap();
// Drain-on-contact: touching g1 detects the death and sweeps the pid // Gone from every group it joined, not just one.
// out of every group (g2 included), not just g1. assert!(members("g1").is_empty(), "gone from g1");
assert!(members("g1").is_empty(), "evicted from the touched group"); assert!(members("g2").is_empty(), "and from g2");
assert!(
members("g2").is_empty(),
"and swept from the untouched group"
);
assert_eq!(pick("g1"), None); assert_eq!(pick("g1"), None);
}); });
} }
@@ -86,11 +83,7 @@ fn live_members_survive_a_peers_death() {
tx_a.send(()).unwrap(); tx_a.send(()).unwrap();
a.join().unwrap(); a.join().unwrap();
assert_eq!( assert_eq!(members("svc"), vec![b.pid()], "only the dead peer is gone");
members("svc"),
vec![b.pid()],
"only the dead peer is reaped"
);
assert_eq!(pick("svc"), Some(b.pid())); assert_eq!(pick("svc"), Some(b.pid()));
tx_b.send(()).unwrap(); tx_b.send(()).unwrap();
@@ -123,18 +116,18 @@ fn leave_drops_a_membership_without_affecting_others() {
} }
#[test] #[test]
fn joining_an_already_dead_pid_is_evicted_on_next_contact() { fn joining_an_already_dead_pid_never_shows_in_a_read() {
run(|| { run(|| {
let h = spawn(|| {}); let h = spawn(|| {});
let pid = h.pid(); let pid = h.pid();
h.join().unwrap(); // actor is finalized before we join it to anything h.join().unwrap(); // actor is finalized before we join it to anything
// monitor() on a gone pid queues a NoProc Down immediately, so the // join() on a gone pid queues a NoProc Down to the reaper immediately;
// membership is reaped the next time the group is touched. // reads never show it either way (slot-liveness backstop).
join("late", pid); join("late", pid);
assert!( assert!(
members("late").is_empty(), members("late").is_empty(),
"dead-at-join member is reaped on read" "dead-at-join member never reads as live"
); );
assert_eq!(pick("late"), None); assert_eq!(pick("late"), None);
}); });
+43
View File
@@ -258,3 +258,46 @@ fn send_dyn_to_dead_pid_is_dead() {
assert!(matches!(send_dyn::<u64>(p, 1u64), Err(SendError::Dead(_)))); assert!(matches!(send_dyn::<u64>(p, 1u64), Err(SendError::Dead(_))));
}); });
} }
// --- one channel per message type per actor -----------------------------------
/// Registering a second name of the same message type on one actor, with a
/// *fresh* channel, would silently replace and close the first — so it
/// panics (found in RFC 010 c9). The sanctioned shapes stay quiet: bind both
/// names to a clone of one sender, or use two actors.
#[test]
#[should_panic(expected = "already publishes a live channel")]
fn second_live_channel_of_same_type_on_one_actor_panics() {
run(|| {
let (tx1, _rx1) = channel::<u64>();
let (tx2, _rx2) = channel::<u64>();
register(Name::<u64>::new("dup-a"), tx1).unwrap();
register(Name::<u64>::new("dup-b"), tx2).unwrap(); // panics
});
}
#[test]
fn two_names_on_one_cloned_sender_is_fine() {
run(|| {
let (tx, rx) = channel::<u64>();
register(Name::<u64>::new("twin-a"), tx.clone()).unwrap();
register(Name::<u64>::new("twin-b"), tx).unwrap();
send(Name::<u64>::new("twin-a"), 1).unwrap();
send(Name::<u64>::new("twin-b"), 2).unwrap();
assert_eq!(rx.recv().unwrap(), 1);
assert_eq!(rx.recv().unwrap(), 2);
});
}
#[test]
fn replacing_a_channel_whose_receiver_is_gone_is_fine() {
run(|| {
let (tx1, rx1) = channel::<u64>();
register(Name::<u64>::new("reborn"), tx1).unwrap();
drop(rx1); // old inbox gone: replacement is the honest thing to do
let (tx2, rx2) = channel::<u64>();
register(Name::<u64>::new("reborn-2"), tx2).unwrap();
send(Name::<u64>::new("reborn-2"), 9).unwrap();
assert_eq!(rx2.recv().unwrap(), 9);
});
}