280 lines
10 KiB
Rust
280 lines
10 KiB
Rust
//! One-for-one supervisor tests.
|
|
//!
|
|
//! A `OneForOne` runs as an ordinary actor: it spawns its children under
|
|
//! itself, collects their termination `Signal`s on a single mailbox, and
|
|
//! restarts them per policy until either every child is in a terminal,
|
|
//! non-restartable state (then `run()` returns) or the restart intensity cap
|
|
//! is tripped (then `run()` gives up and returns).
|
|
//!
|
|
//! Policies:
|
|
//! - `Permanent` — restart on any termination (normal or panic).
|
|
//! - `Transient` — restart only on panic; a normal exit is "done".
|
|
//! - `Temporary` — never restart.
|
|
//!
|
|
//! All cases below are deterministic under the single-thread runtime: the
|
|
//! supervisor parks in `recv()` while a child runs to completion, so signals
|
|
//! arrive one at a time in a fixed order.
|
|
|
|
use smarm::supervisor::{ChildSpec, OneForOne, Restart, Strategy};
|
|
use smarm::{run, sleep, spawn};
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::Duration;
|
|
|
|
fn counting_child(
|
|
counter: &Arc<AtomicUsize>,
|
|
behavior: impl Fn(usize) + Send + Sync + 'static,
|
|
) -> impl Fn() + Send + Sync + 'static {
|
|
let c = counter.clone();
|
|
move || {
|
|
let n = c.fetch_add(1, Ordering::SeqCst) + 1;
|
|
behavior(n);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn transient_child_is_restarted_on_panic_then_settles() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |n| {
|
|
if n < 3 {
|
|
panic!("crash {n}");
|
|
}
|
|
// 3rd run returns normally -> Transient does not restart.
|
|
});
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(
|
|
runs.load(Ordering::SeqCst),
|
|
3,
|
|
"two restarts then a clean exit"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn transient_child_normal_exit_is_not_restarted() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |_| { /* return normally */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.child(ChildSpec::new(Restart::Transient, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(runs.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn temporary_child_is_not_restarted_on_panic() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |_| panic!("once"));
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.child(ChildSpec::new(Restart::Temporary, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(runs.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn intensity_cap_stops_a_permanent_crash_loop() {
|
|
let runs = Arc::new(AtomicUsize::new(0));
|
|
let r = runs.clone();
|
|
run(move || {
|
|
let child = counting_child(&r, |n| panic!("always {n}"));
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(3, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Permanent, child))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
// 1 initial run + 3 restarts, then the 4th failure trips the cap.
|
|
assert_eq!(runs.load(Ordering::SeqCst), 4);
|
|
}
|
|
|
|
#[test]
|
|
fn one_for_one_restarts_only_the_failed_child() {
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb) = (a.clone(), b.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n < 2 {
|
|
panic!("a crash {n}");
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |_| { /* runs once, normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Temporary, child_b))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 2, "A restarts once then settles");
|
|
assert_eq!(b.load(Ordering::SeqCst), 1, "B is untouched by A's restart");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// one_for_all / rest_for_one (roadmap #2)
|
|
//
|
|
// These exercise the group-restart strategies. The discriminator across all
|
|
// three strategies is how a *sibling* of the failed child is treated:
|
|
// - one_for_one : sibling untouched.
|
|
// - rest_for_one: only siblings started *after* the failed child are cycled.
|
|
// - one_for_all : every sibling is cycled, regardless of its own policy.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn one_for_all_restarts_a_normally_exited_sibling() {
|
|
// A panics once then settles; B always exits normally. Under one_for_all,
|
|
// A's failure cycles the whole group, so B is *restarted even though it had
|
|
// exited normally* and its own Transient policy would never self-restart.
|
|
// That second B run is what distinguishes one_for_all from one_for_one
|
|
// (where B would run exactly once).
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb) = (a.clone(), b.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n < 2 {
|
|
panic!("a crash {n}");
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |_| { /* always normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.strategy(Strategy::OneForAll)
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Transient, child_b))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 2, "A: crash then clean run");
|
|
assert_eq!(
|
|
b.load(Ordering::SeqCst),
|
|
2,
|
|
"B cycled with the group despite a clean exit"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rest_for_one_cycles_only_the_suffix() {
|
|
// Three children A(0) B(1) C(2). B panics once. Under rest_for_one only the
|
|
// children started after B (i.e. C) are cycled with it; A — started before
|
|
// B — is left running untouched.
|
|
//
|
|
// A must still be *alive* when B's failure is processed, otherwise "not
|
|
// restarted" is indistinguishable from "already gone". So A sleeps on its
|
|
// first run (parking across B's failure) and returns immediately on any
|
|
// later run. Result:
|
|
// one_for_one : A=1 B=2 C=1
|
|
// rest_for_one: A=1 B=2 C=2 <- this test
|
|
// one_for_all : A=2 B=2 C=2
|
|
let a = Arc::new(AtomicUsize::new(0));
|
|
let b = Arc::new(AtomicUsize::new(0));
|
|
let c = Arc::new(AtomicUsize::new(0));
|
|
let (ra, rb, rc) = (a.clone(), b.clone(), c.clone());
|
|
run(move || {
|
|
let child_a = counting_child(&ra, |n| {
|
|
if n == 1 {
|
|
// Stay alive (parked) across B's failure, then exit on its own.
|
|
sleep(Duration::from_millis(20));
|
|
}
|
|
});
|
|
let child_b = counting_child(&rb, |n| {
|
|
if n < 2 {
|
|
panic!("b crash {n}");
|
|
}
|
|
});
|
|
let child_c = counting_child(&rc, |_| { /* always normal */ });
|
|
let sup = spawn(move || {
|
|
OneForOne::new()
|
|
.strategy(Strategy::RestForOne)
|
|
.intensity(10, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Transient, child_a))
|
|
.child(ChildSpec::new(Restart::Transient, child_b))
|
|
.child(ChildSpec::new(Restart::Transient, child_c))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
assert_eq!(a.load(Ordering::SeqCst), 1, "A (prefix) is left untouched");
|
|
assert_eq!(b.load(Ordering::SeqCst), 2, "B (the failure) is restarted");
|
|
assert_eq!(c.load(Ordering::SeqCst), 2, "C (suffix) is cycled with B");
|
|
}
|
|
|
|
#[test]
|
|
fn group_restart_and_shutdown_stop_in_reverse_start_order() {
|
|
// A(0) and B(1) park (long sleep) and record their teardown order on Drop;
|
|
// C(2) panics on every run. Under one_for_all with intensity(1):
|
|
// - C's first panic cycles the group: live siblings B,A are stopped in
|
|
// reverse start order -> Drop order [B, A].
|
|
// - after the restart C panics again, tripping the cap; the supervisor
|
|
// shuts down, stopping the survivors B,A in reverse order again.
|
|
// We assert the first teardown wave is [B, A] (reverse of start order A,B).
|
|
let order = Arc::new(Mutex::new(Vec::<&'static str>::new()));
|
|
|
|
struct Rec(&'static str, Arc<Mutex<Vec<&'static str>>>);
|
|
impl Drop for Rec {
|
|
fn drop(&mut self) {
|
|
self.1.lock().unwrap().push(self.0);
|
|
}
|
|
}
|
|
|
|
let oa = order.clone();
|
|
run(move || {
|
|
let o_a = oa.clone();
|
|
let o_b = oa.clone();
|
|
let sup = spawn(move || {
|
|
let o_a2 = o_a.clone();
|
|
let o_b2 = o_b.clone();
|
|
OneForOne::new()
|
|
.strategy(Strategy::OneForAll)
|
|
.intensity(1, Duration::from_secs(60))
|
|
.child(ChildSpec::new(Restart::Permanent, move || {
|
|
let _g = Rec("A", o_a2.clone());
|
|
sleep(Duration::from_secs(30)); // interrupted by the stop
|
|
}))
|
|
.child(ChildSpec::new(Restart::Permanent, move || {
|
|
let _g = Rec("B", o_b2.clone());
|
|
sleep(Duration::from_secs(30));
|
|
}))
|
|
.child(ChildSpec::new(Restart::Permanent, || panic!("c always")))
|
|
.run();
|
|
});
|
|
sup.join().unwrap();
|
|
});
|
|
|
|
let recorded = order.lock().unwrap().clone();
|
|
assert!(
|
|
recorded.len() >= 2,
|
|
"expected at least one teardown wave, got {recorded:?}"
|
|
);
|
|
assert_eq!(
|
|
&recorded[..2],
|
|
&["B", "A"],
|
|
"siblings must be torn down in reverse start order (got {recorded:?})"
|
|
);
|
|
}
|