fix(runtime): don't let orphaned timers block shutdown
A cooperatively-cancelled actor that was sleeping (or in a bounded wait) leaves its timer entry behind in the heap. The scheduler's shutdown check required "no pending timers", so a single orphaned far-future deadline kept the whole runtime alive until it fired — e.g. cancelling an actor mid sleep(30s) hung run() for 30s. A timer can only ever do useful work by waking a live actor, so once no actors are live every remaining entry is orphaned by definition. Drop the pending-timers condition from the shutdown check (live == 0 already implies nothing a timer could wake) and clear the heap on the way out.
This commit is contained in:
+11
-5
@@ -779,11 +779,17 @@ fn schedule_loop(inner: &Arc<RuntimeInner>, slot: usize) {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Queue was empty. Re-examine to decide exit vs wait.
|
// Queue was empty. Re-examine to decide exit vs wait.
|
||||||
// All four conditions must hold simultaneously before we exit:
|
// We exit only when nothing can ever produce more work:
|
||||||
// 1. run queue is still empty
|
// 1. run queue is still empty
|
||||||
// 2. no live actors (nothing parked, nothing mid-finalize)
|
// 2. no live actors (nothing parked, nothing mid-finalize)
|
||||||
// 3. no pending timers
|
// 3. no outstanding IO
|
||||||
// 4. no outstanding IO
|
// Pending timers are deliberately NOT a condition: a timer can
|
||||||
|
// only ever do something useful by waking a live actor, so once
|
||||||
|
// `live == 0` every remaining timer entry is orphaned (e.g. a
|
||||||
|
// sleeping actor that was cooperatively cancelled out of its
|
||||||
|
// sleep, leaving its deadline behind). Such entries must not
|
||||||
|
// keep the runtime alive until they fire — that could be an
|
||||||
|
// arbitrarily long hang. They are dropped here on the way out.
|
||||||
let next = s.timers.peek_deadline();
|
let next = s.timers.peek_deadline();
|
||||||
let (out, fd) = match s.io.as_ref() {
|
let (out, fd) = match s.io.as_ref() {
|
||||||
Some(io) => (
|
Some(io) => (
|
||||||
@@ -793,9 +799,9 @@ fn schedule_loop(inner: &Arc<RuntimeInner>, slot: usize) {
|
|||||||
None => (0, None),
|
None => (0, None),
|
||||||
};
|
};
|
||||||
let live = s.slots.iter().filter(|slot| slot.actor.is_some()).count();
|
let live = s.slots.iter().filter(|slot| slot.actor.is_some()).count();
|
||||||
let all_clear = s.run_queue.is_empty() && live == 0
|
let all_clear = s.run_queue.is_empty() && live == 0 && out == 0;
|
||||||
&& next.is_none() && out == 0;
|
|
||||||
if all_clear {
|
if all_clear {
|
||||||
|
s.timers.clear();
|
||||||
PopResult::AllDone
|
PopResult::AllDone
|
||||||
} else {
|
} else {
|
||||||
PopResult::Idle { next_deadline: next, io_outstanding: out, wake_fd: fd }
|
PopResult::Idle { next_deadline: next, io_outstanding: out, wake_fd: fd }
|
||||||
|
|||||||
@@ -119,6 +119,13 @@ impl Timers {
|
|||||||
self.heap.is_empty()
|
self.heap.is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drop all pending entries. Called by the scheduler when it has decided
|
||||||
|
/// no actor is live: any remaining timer is orphaned and exists only to be
|
||||||
|
/// discarded so it can't keep the runtime alive.
|
||||||
|
pub fn clear(&mut self) {
|
||||||
|
self.heap.clear();
|
||||||
|
}
|
||||||
|
|
||||||
/// Soonest pending deadline, or `None` if the heap is empty.
|
/// Soonest pending deadline, or `None` if the heap is empty.
|
||||||
pub fn peek_deadline(&self) -> Option<Instant> {
|
pub fn peek_deadline(&self) -> Option<Instant> {
|
||||||
self.heap.peek().map(|r| r.0.deadline)
|
self.heap.peek().map(|r| r.0.deadline)
|
||||||
|
|||||||
Reference in New Issue
Block a user