smarm-dev
68c7c96749
Adds a Strategy enum (OneForOne default, OneForAll, RestForOne) selected via OneForOne::strategy(). The triggering child's Restart policy still decides whether anything restarts; the strategy decides which siblings are cycled with it: - OneForAll : all live siblings - RestForOne: siblings started after the failed child Group restarts stop the affected survivors cooperatively (request_stop) in reverse start order, await each one's termination signal on the existing funnel (no new channel, no select), then restart the whole set in start order. Signals that arrive for a child we aren't currently stopping are stashed and replayed by the main loop. A Stopped signal now counts as abnormal for the restart decision. On giving up (intensity cap) or any exit with survivors, an ordered shutdown stops the remaining children in reverse start order instead of leaking them; on the normal all-settled exit it's a no-op. The struct keeps the OneForOne name for compatibility (existing tests unchanged); rename is a later refactor.
272 lines
10 KiB
Rust
272 lines
10 KiB
Rust
//! One-for-one supervisor tests.
|
|
//!
|
|
//! A `OneForOne` runs as an ordinary actor: it spawns its children under
|
|
//! itself, collects their termination `Signal`s on a single mailbox, and
|
|
//! restarts them per policy until either every child is in a terminal,
|
|
//! non-restartable state (then `run()` returns) or the restart intensity cap
|
|
//! is tripped (then `run()` gives up and returns).
|
|
//!
|
|
//! Policies:
|
|
//! - `Permanent` — restart on any termination (normal or panic).
|
|
//! - `Transient` — restart only on panic; a normal exit is "done".
|
|
//! - `Temporary` — never restart.
|
|
//!
|
|
//! All cases below are deterministic under the single-thread runtime: the
|
|
//! supervisor parks in `recv()` while a child runs to completion, so signals
|
|
//! arrive one at a time in a fixed order.
|
|
|
|
use smarm::supervisor::{ChildSpec, OneForOne, Restart, Strategy};
|
|
use smarm::{run, sleep, spawn};
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
fn counting_child(
|
|
counter: &Arc<AtomicUsize>,
|
|
behavior: impl Fn(usize) + Send + Sync + 'static,
|
|
) -> impl Fn() + Send + Sync + 'static {
|
|
let c = counter.clone();
|
|
move || {
|
|
let n = c.fetch_add(1, Ordering::SeqCst) + 1;
|
|
behavior(n);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn transient_child_is_restarted_on_panic_then_settles() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |n| {
|
|
if n < 3 {
|
|
panic!("crash {n}");
|
|
}
|
|
// 3rd run returns normally -> Transient does not restart.
|
|
});
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(runs.load(Ordering::SeqCst), 3, "two restarts then a clean exit");
|
|
}
|
|
|
|
#[test]
|
|
fn transient_child_normal_exit_is_not_restarted() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |_| { /* return normally */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.child(ChildSpec::new(Restart::Transient, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(runs.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn temporary_child_is_not_restarted_on_panic() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |_| panic!("once"));
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.child(ChildSpec::new(Restart::Temporary, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(runs.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn intensity_cap_stops_a_permanent_crash_loop() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |n| panic!("always {n}"));
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(3, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Permanent, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
// 1 initial run + 3 restarts, then the 4th failure trips the cap.
|
|
assert_eq!(runs.load(Ordering::SeqCst), 4);
|
|
}
|
|
|
|
#[test]
|
|
fn one_for_one_restarts_only_the_failed_child() {
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb) = (a.clone(), b.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n < 2 {
|
|
panic!("a crash {n}");
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |_| { /* runs once, normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Temporary, child_b))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 2, "A restarts once then settles");
|
|
assert_eq!(b.load(Ordering::SeqCst), 1, "B is untouched by A's restart");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// one_for_all / rest_for_one (roadmap #2)
|
|
//
|
|
// These exercise the group-restart strategies. The discriminator across all
|
|
// three strategies is how a *sibling* of the failed child is treated:
|
|
// - one_for_one : sibling untouched.
|
|
// - rest_for_one: only siblings started *after* the failed child are cycled.
|
|
// - one_for_all : every sibling is cycled, regardless of its own policy.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn one_for_all_restarts_a_normally_exited_sibling() {
|
|
// A panics once then settles; B always exits normally. Under one_for_all,
|
|
// A's failure cycles the whole group, so B is *restarted even though it had
|
|
// exited normally* and its own Transient policy would never self-restart.
|
|
// That second B run is what distinguishes one_for_all from one_for_one
|
|
// (where B would run exactly once).
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb) = (a.clone(), b.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n < 2 {
|
|
panic!("a crash {n}");
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |_| { /* always normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.strategy(Strategy::OneForAll)
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Transient, child_b))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 2, "A: crash then clean run");
|
|
assert_eq!(b.load(Ordering::SeqCst), 2, "B cycled with the group despite a clean exit");
|
|
}
|
|
|
|
#[test]
|
|
fn rest_for_one_cycles_only_the_suffix() {
|
|
// Three children A(0) B(1) C(2). B panics once. Under rest_for_one only the
|
|
// children started after B (i.e. C) are cycled with it; A — started before
|
|
// B — is left running untouched.
|
|
//
|
|
// A must still be *alive* when B's failure is processed, otherwise "not
|
|
// restarted" is indistinguishable from "already gone". So A sleeps on its
|
|
// first run (parking across B's failure) and returns immediately on any
|
|
// later run. Result:
|
|
// one_for_one : A=1 B=2 C=1
|
|
// rest_for_one: A=1 B=2 C=2 <- this test
|
|
// one_for_all : A=2 B=2 C=2
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let c = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb, rc) = (a.clone(), b.clone(), c.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n == 1 {
|
|
// Stay alive (parked) across B's failure, then exit on its own.
|
|
sleep(Duration::from_millis(20));
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |n| {
|
|
if n < 2 {
|
|
panic!("b crash {n}");
|
|
}
|
|
});
|
|
let child_c = counting_child(&rc, |_| { /* always normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.strategy(Strategy::RestForOne)
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Transient, child_b))
|
|
.child(ChildSpec::new(Restart::Transient, child_c))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 1, "A (prefix) is left untouched");
|
|
assert_eq!(b.load(Ordering::SeqCst), 2, "B (the failure) is restarted");
|
|
assert_eq!(c.load(Ordering::SeqCst), 2, "C (suffix) is cycled with B");
|
|
}
|
|
|
|
#[test]
|
|
fn group_restart_and_shutdown_stop_in_reverse_start_order() {
|
|
// A(0) and B(1) park (long sleep) and record their teardown order on Drop;
|
|
// C(2) panics on every run. Under one_for_all with intensity(1):
|
|
// - C's first panic cycles the group: live siblings B,A are stopped in
|
|
// reverse start order -> Drop order [B, A].
|
|
// - after the restart C panics again, tripping the cap; the supervisor
|
|
// shuts down, stopping the survivors B,A in reverse order again.
|
|
// We assert the first teardown wave is [B, A] (reverse of start order A,B).
|
|
let order = Arc::new(Mutex::new(Vec::<&'static str>::new()));
|
|
|
|
struct Rec(&'static str, Arc<Mutex<Vec<&'static str>>>);
|
|
impl Drop for Rec {
|
|
fn drop(&mut self) {
|
|
self.1.lock().unwrap().push(self.0);
|
|
}
|
|
}
|
|
|
|
let oa = order.clone();
|
|
run(move || {
|
|
let o_a = oa.clone();
|
|
let o_b = oa.clone();
|
|
let sup = spawn(move || {
|
|
let o_a2 = o_a.clone();
|
|
let o_b2 = o_b.clone();
|
|
OneForOne::new()
|
|
.strategy(Strategy::OneForAll)
|
|
.intensity(1, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Permanent, move || {
|
|
let _g = Rec("A", o_a2.clone());
|
|
sleep(Duration::from_secs(30)); // interrupted by the stop
|
|
}))
|
|
.child(ChildSpec::new(Restart::Permanent, move || {
|
|
let _g = Rec("B", o_b2.clone());
|
|
sleep(Duration::from_secs(30));
|
|
}))
|
|
.child(ChildSpec::new(Restart::Permanent, || panic!("c always")))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
|
|
let recorded = order.lock().unwrap().clone();
|
|
assert!(
|
|
recorded.len() >= 2,
|
|
"expected at least one teardown wave, got {recorded:?}"
|
|
);
|
|
assert_eq!(
|
|
&recorded[..2],
|
|
&["B", "A"],
|
|
"siblings must be torn down in reverse start order (got {recorded:?})"
|
|
);
|
|
}
|