feat(supervisor): one_for_all and rest_for_one strategies + ordered shutdown
Adds a Strategy enum (OneForOne default, OneForAll, RestForOne) selected via OneForOne::strategy(). The triggering child's Restart policy still decides whether anything restarts; the strategy decides which siblings are cycled with it: - OneForAll : all live siblings - RestForOne: siblings started after the failed child Group restarts stop the affected survivors cooperatively (request_stop) in reverse start order, await each one's termination signal on the existing funnel (no new channel, no select), then restart the whole set in start order. Signals that arrive for a child we aren't currently stopping are stashed and replayed by the main loop. A Stopped signal now counts as abnormal for the restart decision. On giving up (intensity cap) or any exit with survivors, an ordered shutdown stops the remaining children in reverse start order instead of leaking them; on the normal all-settled exit it's a no-op. The struct keeps the OneForOne name for compatibility (existing tests unchanged); rename is a later refactor.
This commit is contained in:
+144
-3
@@ -15,10 +15,10 @@
|
||||
//! supervisor parks in `recv()` while a child runs to completion, so signals
|
||||
//! arrive one at a time in a fixed order.
|
||||
|
||||
use smarm::supervisor::{ChildSpec, OneForOne, Restart};
|
||||
use smarm::{run, spawn};
|
||||
use smarm::supervisor::{ChildSpec, OneForOne, Restart, Strategy};
|
||||
use smarm::{run, sleep, spawn};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
fn counting_child(
|
||||
@@ -128,3 +128,144 @@ fn one_for_one_restarts_only_the_failed_child() {
|
||||
assert_eq!(a.load(Ordering::SeqCst), 2, "A restarts once then settles");
|
||||
assert_eq!(b.load(Ordering::SeqCst), 1, "B is untouched by A's restart");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// one_for_all / rest_for_one (roadmap #2)
|
||||
//
|
||||
// These exercise the group-restart strategies. The discriminator across all
|
||||
// three strategies is how a *sibling* of the failed child is treated:
|
||||
// - one_for_one : sibling untouched.
|
||||
// - rest_for_one: only siblings started *after* the failed child are cycled.
|
||||
// - one_for_all : every sibling is cycled, regardless of its own policy.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn one_for_all_restarts_a_normally_exited_sibling() {
|
||||
// A panics once then settles; B always exits normally. Under one_for_all,
|
||||
// A's failure cycles the whole group, so B is *restarted even though it had
|
||||
// exited normally* and its own Transient policy would never self-restart.
|
||||
// That second B run is what distinguishes one_for_all from one_for_one
|
||||
// (where B would run exactly once).
|
||||
let a = Arc::new(AtomicUsize::new(0));
|
||||
let b = Arc::new(AtomicUsize::new(0));
|
||||
let (ra, rb) = (a.clone(), b.clone());
|
||||
run(move || {
|
||||
let child_a = counting_child(&ra, |n| {
|
||||
if n < 2 {
|
||||
panic!("a crash {n}");
|
||||
}
|
||||
});
|
||||
let child_b = counting_child(&rb, |_| { /* always normal */ });
|
||||
let sup = spawn(move || {
|
||||
OneForOne::new()
|
||||
.strategy(Strategy::OneForAll)
|
||||
.intensity(10, Duration::from_secs(60))
|
||||
.child(ChildSpec::new(Restart::Transient, child_a))
|
||||
.child(ChildSpec::new(Restart::Transient, child_b))
|
||||
.run();
|
||||
});
|
||||
sup.join().unwrap();
|
||||
});
|
||||
assert_eq!(a.load(Ordering::SeqCst), 2, "A: crash then clean run");
|
||||
assert_eq!(b.load(Ordering::SeqCst), 2, "B cycled with the group despite a clean exit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rest_for_one_cycles_only_the_suffix() {
|
||||
// Three children A(0) B(1) C(2). B panics once. Under rest_for_one only the
|
||||
// children started after B (i.e. C) are cycled with it; A — started before
|
||||
// B — is left running untouched.
|
||||
//
|
||||
// A must still be *alive* when B's failure is processed, otherwise "not
|
||||
// restarted" is indistinguishable from "already gone". So A sleeps on its
|
||||
// first run (parking across B's failure) and returns immediately on any
|
||||
// later run. Result:
|
||||
// one_for_one : A=1 B=2 C=1
|
||||
// rest_for_one: A=1 B=2 C=2 <- this test
|
||||
// one_for_all : A=2 B=2 C=2
|
||||
let a = Arc::new(AtomicUsize::new(0));
|
||||
let b = Arc::new(AtomicUsize::new(0));
|
||||
let c = Arc::new(AtomicUsize::new(0));
|
||||
let (ra, rb, rc) = (a.clone(), b.clone(), c.clone());
|
||||
run(move || {
|
||||
let child_a = counting_child(&ra, |n| {
|
||||
if n == 1 {
|
||||
// Stay alive (parked) across B's failure, then exit on its own.
|
||||
sleep(Duration::from_millis(20));
|
||||
}
|
||||
});
|
||||
let child_b = counting_child(&rb, |n| {
|
||||
if n < 2 {
|
||||
panic!("b crash {n}");
|
||||
}
|
||||
});
|
||||
let child_c = counting_child(&rc, |_| { /* always normal */ });
|
||||
let sup = spawn(move || {
|
||||
OneForOne::new()
|
||||
.strategy(Strategy::RestForOne)
|
||||
.intensity(10, Duration::from_secs(60))
|
||||
.child(ChildSpec::new(Restart::Transient, child_a))
|
||||
.child(ChildSpec::new(Restart::Transient, child_b))
|
||||
.child(ChildSpec::new(Restart::Transient, child_c))
|
||||
.run();
|
||||
});
|
||||
sup.join().unwrap();
|
||||
});
|
||||
assert_eq!(a.load(Ordering::SeqCst), 1, "A (prefix) is left untouched");
|
||||
assert_eq!(b.load(Ordering::SeqCst), 2, "B (the failure) is restarted");
|
||||
assert_eq!(c.load(Ordering::SeqCst), 2, "C (suffix) is cycled with B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn group_restart_and_shutdown_stop_in_reverse_start_order() {
|
||||
// A(0) and B(1) park (long sleep) and record their teardown order on Drop;
|
||||
// C(2) panics on every run. Under one_for_all with intensity(1):
|
||||
// - C's first panic cycles the group: live siblings B,A are stopped in
|
||||
// reverse start order -> Drop order [B, A].
|
||||
// - after the restart C panics again, tripping the cap; the supervisor
|
||||
// shuts down, stopping the survivors B,A in reverse order again.
|
||||
// We assert the first teardown wave is [B, A] (reverse of start order A,B).
|
||||
let order = Arc::new(Mutex::new(Vec::<&'static str>::new()));
|
||||
|
||||
struct Rec(&'static str, Arc<Mutex<Vec<&'static str>>>);
|
||||
impl Drop for Rec {
|
||||
fn drop(&mut self) {
|
||||
self.1.lock().unwrap().push(self.0);
|
||||
}
|
||||
}
|
||||
|
||||
let oa = order.clone();
|
||||
run(move || {
|
||||
let o_a = oa.clone();
|
||||
let o_b = oa.clone();
|
||||
let sup = spawn(move || {
|
||||
let o_a2 = o_a.clone();
|
||||
let o_b2 = o_b.clone();
|
||||
OneForOne::new()
|
||||
.strategy(Strategy::OneForAll)
|
||||
.intensity(1, Duration::from_secs(60))
|
||||
.child(ChildSpec::new(Restart::Permanent, move || {
|
||||
let _g = Rec("A", o_a2.clone());
|
||||
sleep(Duration::from_secs(30)); // interrupted by the stop
|
||||
}))
|
||||
.child(ChildSpec::new(Restart::Permanent, move || {
|
||||
let _g = Rec("B", o_b2.clone());
|
||||
sleep(Duration::from_secs(30));
|
||||
}))
|
||||
.child(ChildSpec::new(Restart::Permanent, || panic!("c always")))
|
||||
.run();
|
||||
});
|
||||
sup.join().unwrap();
|
||||
});
|
||||
|
||||
let recorded = order.lock().unwrap().clone();
|
||||
assert!(
|
||||
recorded.len() >= 2,
|
||||
"expected at least one teardown wave, got {recorded:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
&recorded[..2],
|
||||
&["B", "A"],
|
||||
"siblings must be torn down in reverse start order (got {recorded:?})"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user