refactor(wakes): epoch-stamp every registration-based wake; retire per-primitive wait seqs

The slot epoch is THE wait identity, so the hand-rolled per-primitive copies
go away: channel loses cur_wait/next_wait_seq/timed_out, mutex loses Wait.seq
and next_seq, TimerTarget::on_timeout takes the epoch. Registrations become
(pid, epoch) — channel parked_receiver, mutex waiters, io fd waiters,
Blocking io completions, sleep timers, joiner lists — and their wakers move
to unpark_at. begin_wait is lock-free, so each primitive opens the wait
inside the same critical section that publishes the registration.

recv_timeout's wake-classification loop collapses: wakes are precise, so
queue → Ok, senders==0 → Disconnected, else Timeout — the 'Defensive'
re-register branch is now unreachable by protocol, not by audit. Same for
Mutex::lock_timeout's one-shot park.

End-state invariant, auditable in one sentence: the only wildcard wake is
request_stop, which is terminal.
This commit is contained in:
smarm
2026-06-10 07:15:29 +00:00
parent 4913835c02
commit 400854ac5d
7 changed files with 185 additions and 147 deletions
+55 -67
View File
@@ -39,27 +39,22 @@ pub fn channel<T>() -> (Sender<T>, Receiver<T>) {
parked_receiver: None, parked_receiver: None,
senders: 1, senders: 1,
receiver_alive: true, receiver_alive: true,
cur_wait: None,
next_wait_seq: 0,
timed_out: false,
})); }));
(Sender { inner: inner.clone() }, Receiver { inner }) (Sender { inner: inner.clone() }, Receiver { inner })
} }
struct Inner<T> { struct Inner<T> {
queue: VecDeque<T>, queue: VecDeque<T>,
parked_receiver: Option<Pid>, /// The parked receiver's `(pid, park-epoch)`. The epoch is the slot
/// word's runtime-wide wait identity (see slot_state.rs): wakers call
/// `unpark_at(pid, epoch)`, so an entry left over from an already-woken
/// wait — a `select` loser arm, a satisfied `recv_timeout`'s timer — is
/// inert: the wake fails the word's epoch CAS and no-ops. This replaces
/// the old per-channel `cur_wait`/`next_wait_seq`/`timed_out` trio: wait
/// identity now exists exactly once, in the slot word.
parked_receiver: Option<(Pid, u32)>,
senders: usize, senders: usize,
receiver_alive: bool, receiver_alive: bool,
/// The seq of the receiver's current *bounded* wait (`recv_timeout`), or
/// `None` outside one. Lets `on_timeout` tell "this wait" apart from "a
/// later wait on the same channel" — same convention as `Mutex`.
cur_wait: Option<u64>,
/// Monotonic source for `cur_wait` seqs.
next_wait_seq: u64,
/// Set by `on_timeout` when it cancels the current bounded wait; consumed
/// by the woken receiver to distinguish a timeout from a message wake.
timed_out: bool,
} }
pub struct Sender<T> { pub struct Sender<T> {
@@ -129,8 +124,8 @@ impl<T> Drop for Sender<T> {
None None
} }
}; };
if let Some(pid) = unpark { if let Some((pid, epoch)) = unpark {
crate::scheduler::unpark(pid); crate::scheduler::unpark_at(pid, epoch);
} }
} }
} }
@@ -151,9 +146,9 @@ impl<T> Sender<T> {
g.queue.push_back(value); g.queue.push_back(value);
g.parked_receiver.take() g.parked_receiver.take()
}; };
if let Some(pid) = unpark { if let Some((pid, epoch)) = unpark {
crate::te!(crate::trace::Event::Send { sender: crate::actor::current_pid().unwrap_or(crate::pid::Pid::new(u32::MAX, u32::MAX)), receiver: Some(pid) }); crate::te!(crate::trace::Event::Send { sender: crate::actor::current_pid().unwrap_or(crate::pid::Pid::new(u32::MAX, u32::MAX)), receiver: Some(pid) });
crate::scheduler::unpark(pid); crate::scheduler::unpark_at(pid, epoch);
} else { } else {
crate::te!(crate::trace::Event::Send { sender: crate::actor::current_pid().unwrap_or(crate::pid::Pid::new(u32::MAX, u32::MAX)), receiver: None }); crate::te!(crate::trace::Event::Send { sender: crate::actor::current_pid().unwrap_or(crate::pid::Pid::new(u32::MAX, u32::MAX)), receiver: None });
} }
@@ -178,7 +173,10 @@ impl<T> Receiver<T> {
g.parked_receiver.is_none(), g.parked_receiver.is_none(),
"channel has more than one receiver" "channel has more than one receiver"
); );
g.parked_receiver = Some(me); // begin_wait is lock-free — legal under the Channel lock;
// registering in the same critical section makes the epoch
// atomic with the senders' view of the registration.
g.parked_receiver = Some((me, crate::scheduler::begin_wait()));
crate::te!(crate::trace::Event::RecvPark(me)); crate::te!(crate::trace::Event::RecvPark(me));
} }
// Release the lock before parking — the unparker will need it. // Release the lock before parking — the unparker will need it.
@@ -192,14 +190,21 @@ impl<T> Receiver<T> {
/// `timeout` has elapsed, returning [`RecvTimeoutError::Timeout`]. /// `timeout` has elapsed, returning [`RecvTimeoutError::Timeout`].
/// ///
/// Built on the same timer machinery as `Mutex::lock_timeout`: the wait /// Built on the same timer machinery as `Mutex::lock_timeout`: the wait
/// registers a `WaitTimeout` entry tagged with a per-wait seq; on expiry /// registers a `WaitTimeout` entry stamped with the wait's park-epoch;
/// the channel (as the [`TimerTarget`](crate::timer::TimerTarget)) checks /// on expiry the channel (as the
/// whether *this* wait is still parked and, only then, cancels it. A wake /// [`TimerTarget`](crate::timer::TimerTarget)) checks whether *this*
/// that races the deadline resolves message-first: if a message is /// wait is still parked and, only then, cancels it. A wake that races
/// available when the receiver runs, it is delivered even if the timer /// the deadline resolves message-first: if a message is available when
/// had already fired. A satisfied or abandoned wait leaves its timer /// the receiver runs, it is delivered even if the timer had already
/// entry to expire as a no-op (seq mismatch), per the no-cancellation /// fired. A satisfied or abandoned wait leaves its timer entry to expire
/// convention in `timer.rs`. /// as a no-op (registration gone; epoch consumed), per the
/// no-cancellation convention in `timer.rs`.
///
/// The wake is classified from state alone — wakes are precise (the only
/// stamped wakers of this wait are a send, the last-sender drop, and the
/// timer; a stop wake unwinds out of `park_current` and never reaches
/// the classification), so: message queued → `Ok`; `senders == 0` →
/// `Disconnected`; neither → it was the timer → `Timeout`.
/// ///
/// `Duration::ZERO` is a valid timeout: it parks until the immediately- /// `Duration::ZERO` is a valid timeout: it parks until the immediately-
/// due timer is drained, then reports `Timeout` unless a message was /// due timer is drained, then reports `Timeout` unless a message was
@@ -212,7 +217,7 @@ impl<T> Receiver<T> {
.expect("recv_timeout() called outside an actor"); .expect("recv_timeout() called outside an actor");
// Fast path + wait registration, one critical section. // Fast path + wait registration, one critical section.
let seq; let epoch;
{ {
let mut g = self.inner.lock(); let mut g = self.inner.lock();
if let Some(v) = g.queue.pop_front() { if let Some(v) = g.queue.pop_front() {
@@ -225,11 +230,8 @@ impl<T> Receiver<T> {
g.parked_receiver.is_none(), g.parked_receiver.is_none(),
"channel has more than one receiver" "channel has more than one receiver"
); );
seq = g.next_wait_seq; epoch = crate::scheduler::begin_wait();
g.next_wait_seq = g.next_wait_seq.wrapping_add(1); g.parked_receiver = Some((me, epoch));
g.cur_wait = Some(seq);
g.timed_out = false;
g.parked_receiver = Some(me);
crate::te!(crate::trace::Event::RecvPark(me)); crate::te!(crate::trace::Event::RecvPark(me));
} }
@@ -239,34 +241,18 @@ impl<T> Receiver<T> {
// protocol makes the park below return immediately in that case. // protocol makes the park below return immediately in that case.
let deadline = crate::timer::deadline_from_now(timeout); let deadline = crate::timer::deadline_from_now(timeout);
let target: std::sync::Arc<dyn crate::timer::TimerTarget> = self.inner.clone(); let target: std::sync::Arc<dyn crate::timer::TimerTarget> = self.inner.clone();
crate::scheduler::insert_wait_timer(deadline, me, target, seq); crate::scheduler::insert_wait_timer(deadline, me, target, epoch);
loop { crate::scheduler::park_current();
crate::scheduler::park_current(); crate::te!(crate::trace::Event::RecvWake(crate::actor::current_pid().unwrap()));
crate::te!(crate::trace::Event::RecvWake(crate::actor::current_pid().unwrap())); let mut g = self.inner.lock();
let mut g = self.inner.lock(); if let Some(v) = g.queue.pop_front() {
if let Some(v) = g.queue.pop_front() { return Ok(v);
g.cur_wait = None;
g.timed_out = false;
return Ok(v);
}
if g.senders == 0 {
g.cur_wait = None;
g.timed_out = false;
return Err(RecvTimeoutError::Disconnected);
}
if g.timed_out {
g.cur_wait = None;
g.timed_out = false;
return Err(RecvTimeoutError::Timeout);
}
// Defensive: no current wake source reaches here (a send leaves a
// message, closure zeroes senders, the timer sets timed_out, and
// a stop unwinds out of park_current) — but if one ever does,
// re-register and keep waiting; cur_wait == Some(seq) keeps the
// already-armed timer valid for this wait.
g.parked_receiver = Some(me);
} }
if g.senders == 0 {
return Err(RecvTimeoutError::Disconnected);
}
Err(RecvTimeoutError::Timeout)
} }
/// Selective receive: remove and return the first queued message for which /// Selective receive: remove and return the first queued message for which
@@ -300,7 +286,7 @@ impl<T> Receiver<T> {
g.parked_receiver.is_none(), g.parked_receiver.is_none(),
"channel has more than one receiver" "channel has more than one receiver"
); );
g.parked_receiver = Some(me); g.parked_receiver = Some((me, crate::scheduler::begin_wait()));
crate::te!(crate::trace::Event::RecvPark(me)); crate::te!(crate::trace::Event::RecvPark(me));
} }
// Release the lock before parking — the unparker will need it. // Release the lock before parking — the unparker will need it.
@@ -346,16 +332,18 @@ impl<T> Receiver<T> {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
impl<T: Send + 'static> crate::timer::TimerTarget for RawMutex<Inner<T>> { impl<T: Send + 'static> crate::timer::TimerTarget for RawMutex<Inner<T>> {
fn on_timeout(&self, pid: Pid, wait_seq: u64) { fn on_timeout(&self, pid: Pid, epoch: u32) {
// Cancel the wait only if THIS wait (seq match) is still parked. If a // Cancel the wait only if THIS wait (epoch match) is still
// sender already took `parked_receiver`, the receiver is waking with // registered. If a sender already took `parked_receiver`, the
// a message — message wins, the timer no-ops. If `cur_wait` moved on, // receiver is waking with a message — message wins, the timer
// this entry is stale (the wait was satisfied or abandoned) — no-op. // no-ops. If a later wait by the same receiver is registered, the
// epoch mismatches — stale entry, no-op. (The unpark_at would fail
// its word CAS in either case anyway; checking under the lock keeps
// the registration bookkeeping exact.)
let unpark = { let unpark = {
let mut g = self.lock(); let mut g = self.lock();
if g.cur_wait == Some(wait_seq) && g.parked_receiver == Some(pid) { if g.parked_receiver == Some((pid, epoch)) {
g.parked_receiver = None; g.parked_receiver = None;
g.timed_out = true;
true true
} else { } else {
false false
@@ -364,7 +352,7 @@ impl<T: Send + 'static> crate::timer::TimerTarget for RawMutex<Inner<T>> {
// Unpark outside the channel lock — it may take the run-queue lock; // Unpark outside the channel lock — it may take the run-queue lock;
// legal under a Channel lock, but pointless to nest. // legal under a Channel lock, but pointless to nest.
if unpark { if unpark {
crate::scheduler::unpark(pid); crate::scheduler::unpark_at(pid, epoch);
} }
} }
} }
+11 -7
View File
@@ -84,6 +84,9 @@ use std::thread::JoinHandle as OsJoinHandle;
pub type IoResult = Result<Box<dyn Any + Send>, Box<dyn Any + Send>>; pub type IoResult = Result<Box<dyn Any + Send>, Box<dyn Any + Send>>;
struct Request { struct Request {
/// The submitter's park-epoch — carried through to the `Blocking`
/// completion so the wake is epoch-matched.
epoch: u32,
pid: Pid, pid: Pid,
/// The work to perform. Returns the wire-form result directly. /// The work to perform. Returns the wire-form result directly.
work: Box<dyn FnOnce() -> IoResult + Send>, work: Box<dyn FnOnce() -> IoResult + Send>,
@@ -93,7 +96,7 @@ struct Request {
pub enum Completion { pub enum Completion {
/// A `block_on_io` closure has finished (Ok = return value, Err = panic /// A `block_on_io` closure has finished (Ok = return value, Err = panic
/// payload). /// payload).
Blocking { pid: Pid, result: IoResult }, Blocking { pid: Pid, epoch: u32, result: IoResult },
/// An fd registered via `wait_readable`/`wait_writable` is ready. The /// An fd registered via `wait_readable`/`wait_writable` is ready. The
/// scheduler looks up the parked pid in `waiters`, unparks it, and /// scheduler looks up the parked pid in `waiters`, unparks it, and
/// removes the entry. `pid` isn't in this variant because the epoll /// removes the entry. `pid` isn't in this variant because the epoll
@@ -131,7 +134,7 @@ pub struct IoThread {
/// One parked actor per registered fd. Populated by `wait_readable` / /// One parked actor per registered fd. Populated by `wait_readable` /
/// `wait_writable` and drained by the scheduler when a `FdReady` /// `wait_writable` and drained by the scheduler when a `FdReady`
/// completion is processed. /// completion is processed.
pub waiters: HashMap<RawFd, Pid>, pub waiters: HashMap<RawFd, (Pid, u32)>,
// ----- Threads ----- // ----- Threads -----
@@ -231,12 +234,12 @@ impl IoThread {
} }
/// Hand a request to the pool. Increments `outstanding`. /// Hand a request to the pool. Increments `outstanding`.
pub fn submit(&mut self, pid: Pid, work: Box<dyn FnOnce() -> IoResult + Send>) { pub fn submit(&mut self, pid: Pid, epoch: u32, work: Box<dyn FnOnce() -> IoResult + Send>) {
self.outstanding += 1; self.outstanding += 1;
// Send can only fail if the pool has hung up, which only happens // Send can only fail if the pool has hung up, which only happens
// on shutdown. submit during shutdown is a bug. // on shutdown. submit during shutdown is a bug.
self.tx self.tx
.send(Request { pid, work }) .send(Request { pid, epoch, work })
.expect("io pool hung up unexpectedly"); .expect("io pool hung up unexpectedly");
} }
@@ -265,6 +268,7 @@ impl IoThread {
&mut self, &mut self,
fd: RawFd, fd: RawFd,
pid: Pid, pid: Pid,
epoch: u32,
readable: bool, readable: bool,
writable: bool, writable: bool,
) -> io::Result<()> { ) -> io::Result<()> {
@@ -303,7 +307,7 @@ impl IoThread {
if r < 0 { if r < 0 {
return Err(io::Error::last_os_error()); return Err(io::Error::last_os_error());
} }
self.waiters.insert(fd, pid); self.waiters.insert(fd, (pid, epoch));
Ok(()) Ok(())
} }
@@ -369,7 +373,7 @@ fn pool_loop(
completions: Arc<Mutex<VecDeque<Completion>>>, completions: Arc<Mutex<VecDeque<Completion>>>,
wake_write: RawFd, wake_write: RawFd,
) { ) {
while let Ok(Request { pid, work }) = rx.recv() { while let Ok(Request { pid, epoch, work }) = rx.recv() {
let result: IoResult = match panic::catch_unwind(panic::AssertUnwindSafe(work)) { let result: IoResult = match panic::catch_unwind(panic::AssertUnwindSafe(work)) {
Ok(r) => r, Ok(r) => r,
Err(payload) => Err(payload), Err(payload) => Err(payload),
@@ -377,7 +381,7 @@ fn pool_loop(
completions completions
.lock() .lock()
.unwrap() .unwrap()
.push_back(Completion::Blocking { pid, result }); .push_back(Completion::Blocking { pid, epoch, result });
wake_scheduler(wake_write); wake_scheduler(wake_write);
} }
} }
+24 -18
View File
@@ -33,13 +33,15 @@ impl std::error::Error for LockTimeout {}
struct Wait { struct Wait {
pid: Pid, pid: Pid,
seq: u64, /// The wait's park-epoch (slot-word wait identity, see slot_state.rs).
/// Grants and timeouts wake via `unpark_at(pid, epoch)`; a stale entry
/// can neither be granted by mistake nor wake the wrong wait.
epoch: u32,
} }
struct MutexState { struct MutexState {
holder: Option<Pid>, holder: Option<Pid>,
waiters: VecDeque<Wait>, waiters: VecDeque<Wait>,
next_seq: u64,
default_timeout: Duration, default_timeout: Duration,
} }
@@ -53,7 +55,6 @@ impl MutexCore {
state: StdMutex::new(MutexState { state: StdMutex::new(MutexState {
holder: None, holder: None,
waiters: VecDeque::new(), waiters: VecDeque::new(),
next_seq: 0,
default_timeout, default_timeout,
}), }),
} }
@@ -61,17 +62,17 @@ impl MutexCore {
} }
impl TimerTarget for MutexCore { impl TimerTarget for MutexCore {
fn on_timeout(&self, pid: Pid, wait_seq: u64) { fn on_timeout(&self, pid: Pid, epoch: u32) {
let unpark = { let unpark = {
let mut st = self.state.lock().unwrap(); let mut st = self.state.lock().unwrap();
// Remove from waiters only if still there with matching seq. // Remove from waiters only if still there with matching epoch.
// If the lock was already granted (holder == Some(pid)), the // If the lock was already granted (holder == Some(pid)), the
// timer fired after the grant — treat as no-op; the actor // timer fired after the grant — treat as no-op; the actor
// will see `is_holder == true` and return Ok. // will see `is_holder == true` and return Ok.
if st.holder == Some(pid) { if st.holder == Some(pid) {
return; return;
} }
let pos = st.waiters.iter().position(|w| w.pid == pid && w.seq == wait_seq); let pos = st.waiters.iter().position(|w| w.pid == pid && w.epoch == epoch);
if pos.is_some() { if pos.is_some() {
st.waiters.remove(pos.unwrap()); st.waiters.remove(pos.unwrap());
true true
@@ -80,7 +81,7 @@ impl TimerTarget for MutexCore {
} }
}; };
if unpark { if unpark {
scheduler::unpark(pid); scheduler::unpark_at(pid, epoch);
} }
} }
} }
@@ -133,20 +134,25 @@ impl<T> Mutex<T> {
// Slow path: register as a waiter, set timeout, park. // Slow path: register as a waiter, set timeout, park.
let _np = scheduler::NoPreempt::enter(); let _np = scheduler::NoPreempt::enter();
let seq = { let epoch = {
let mut st = self.core.state.lock().unwrap(); let mut st = self.core.state.lock().unwrap();
let seq = st.next_seq; // begin_wait is lock-free — legal under the state lock; this
st.next_seq = st.next_seq.wrapping_add(1); // makes the epoch atomic with the registration's visibility to
st.waiters.push_back(Wait { pid: me, seq }); // grants and timeouts.
seq let epoch = scheduler::begin_wait();
st.waiters.push_back(Wait { pid: me, epoch });
epoch
}; };
let target: Arc<dyn TimerTarget> = self.core.clone(); let target: Arc<dyn TimerTarget> = self.core.clone();
let deadline = timer::deadline_from_now(timeout); let deadline = timer::deadline_from_now(timeout);
scheduler::insert_wait_timer(deadline, me, target, seq); scheduler::insert_wait_timer(deadline, me, target, epoch);
scheduler::park_current(); scheduler::park_current();
// Resumed. Are we the holder? // Resumed — precisely: only our grant or our timer can wake this
// wait (both epoch-stamped; a stop wake unwinds out of
// park_current). The one-shot interpretation below is therefore
// exhaustive. Are we the holder?
let is_holder = self.core.state.lock().unwrap().holder == Some(me); let is_holder = self.core.state.lock().unwrap().holder == Some(me);
if is_holder { if is_holder {
let value = self.value.lock().unwrap().take() let value = self.value.lock().unwrap().take()
@@ -228,12 +234,12 @@ impl<T> Drop for MutexGuard<'_, T> {
let v = self.value.take().expect("MutexGuard: double drop"); let v = self.value.take().expect("MutexGuard: double drop");
*self.mutex.value.lock().unwrap() = Some(v); *self.mutex.value.lock().unwrap() = Some(v);
let next_pid = { let next = {
let mut st = self.mutex.core.state.lock().unwrap(); let mut st = self.mutex.core.state.lock().unwrap();
match st.waiters.pop_front() { match st.waiters.pop_front() {
Some(w) => { Some(w) => {
st.holder = Some(w.pid); st.holder = Some(w.pid);
Some(w.pid) Some((w.pid, w.epoch))
} }
None => { None => {
st.holder = None; st.holder = None;
@@ -241,8 +247,8 @@ impl<T> Drop for MutexGuard<'_, T> {
} }
} }
}; };
if let Some(pid) = next_pid { if let Some((pid, epoch)) = next {
scheduler::unpark(pid); scheduler::unpark_at(pid, epoch);
} }
} }
} }
+28 -16
View File
@@ -17,7 +17,7 @@
//! } //! }
//! //!
//! Slot { //! Slot {
//! word: AtomicU64 ← (generation << 32) | state — THE state machine //! word: AtomicU64 ← (gen << 32) | (epoch << 8) | state — THE state machine
//! sp: AtomicUsize ← saved stack pointer //! sp: AtomicUsize ← saved stack pointer
//! stop_ptr:AtomicPtr<...> ← into the actor's Arc<AtomicBool> //! stop_ptr:AtomicPtr<...> ← into the actor's Arc<AtomicBool>
//! closure: AtomicPtr<...> ← first-resume closure, swap-to-take //! closure: AtomicPtr<...> ← first-resume closure, swap-to-take
@@ -320,7 +320,9 @@ pub(crate) type Closure = Box<dyn FnOnce() + Send>;
/// link / finalize), never on the yield/park/unpark hot path. /// link / finalize), never on the yield/park/unpark hot path.
pub(crate) struct SlotCold { pub(crate) struct SlotCold {
pub(crate) actor: Option<Actor>, pub(crate) actor: Option<Actor>,
pub(crate) waiters: Vec<Pid>, /// Parked joiners as `(pid, park-epoch)`; finalize wakes each via the
/// epoch-matched unpark.
pub(crate) waiters: Vec<(Pid, u32)>,
pub(crate) outcome: Option<Outcome>, pub(crate) outcome: Option<Outcome>,
pub(crate) supervisor_channel: Option<Sender<Signal>>, pub(crate) supervisor_channel: Option<Sender<Signal>>,
/// Watchers registered via `monitor()`, each tagged with its /// Watchers registered via `monitor()`, each tagged with its
@@ -543,6 +545,14 @@ impl RuntimeInner {
self.unpark_inner(pid, Some(epoch)); self.unpark_inner(pid, Some(epoch));
} }
/// Open a new wait for `pid` (the calling actor itself): bump its
/// park-epoch and return it. See slot_state.rs for the rules.
#[must_use]
pub(crate) fn begin_wait(&self, pid: Pid) -> u32 {
let slot = self.slot_at(pid).expect("begin_wait: own slot vanished");
slot.word.begin_wait(pid.generation())
}
fn unpark_inner(&self, pid: Pid, want: Option<u32>) { fn unpark_inner(&self, pid: Pid, want: Option<u32>) {
if let Some(slot) = self.slot_at(pid) { if let Some(slot) = self.slot_at(pid) {
match slot.word.unpark(pid.generation(), want) { match slot.word.unpark(pid.generation(), want) {
@@ -937,9 +947,9 @@ fn finalize_actor(inner: &Arc<RuntimeInner>, pid: Pid, outcome: Outcome) {
} }
} }
// Unpark joiners. // Unpark joiners (epoch-matched: each registered under the cold lock).
for joiner in waiters { for (joiner, epoch) in waiters {
inner.unpark(joiner); inner.unpark_at(joiner, epoch);
} }
// Reclaim if no outstanding handles (re-verified inside). // Reclaim if no outstanding handles (re-verified inside).
@@ -990,17 +1000,19 @@ fn schedule_loop(inner: &Arc<RuntimeInner>, slot_idx: usize) {
// actor is between `timers.insert_sleep` and // actor is between `timers.insert_sleep` and
// `park_current`; RunningNotified makes the upcoming park // `park_current`; RunningNotified makes the upcoming park
// re-queue), or gone (no-op). // re-queue), or gone (no-op).
crate::timer::Reason::Sleep => inner.unpark(entry.pid), crate::timer::Reason::Sleep { epoch } => {
crate::timer::Reason::WaitTimeout { target, wait_seq } => { inner.unpark_at(entry.pid, epoch)
// The callback may call unpark itself. }
target.on_timeout(entry.pid, wait_seq); crate::timer::Reason::WaitTimeout { target, epoch } => {
// The callback may call unpark_at itself.
target.on_timeout(entry.pid, epoch);
} }
} }
} }
for completion in completions { for completion in completions {
match completion { match completion {
crate::io::Completion::Blocking { pid, result } => { crate::io::Completion::Blocking { pid, epoch, result } => {
if let Some(io) = inner.io.lock().unwrap().as_mut() { if let Some(io) = inner.io.lock().unwrap().as_mut() {
io.outstanding = io.outstanding.saturating_sub(1); io.outstanding = io.outstanding.saturating_sub(1);
} }
@@ -1018,19 +1030,19 @@ fn schedule_loop(inner: &Arc<RuntimeInner>, slot_idx: usize) {
// flight; discard the result. // flight; discard the result.
} }
} }
inner.unpark(pid); inner.unpark_at(pid, epoch);
} }
} }
crate::io::Completion::FdReady { fd, events: _ } => { crate::io::Completion::FdReady { fd, events: _ } => {
// Resolve the parked pid under the io lock, then wake // Resolve the parked pid under the io lock, then wake
// through the protocol. Lock order: io before all. // through the protocol. Lock order: io before all.
let parked_pid = inner.io.lock().unwrap().as_mut().and_then(|io| { let parked = inner.io.lock().unwrap().as_mut().and_then(|io| {
let pid = io.waiters.remove(&fd); let entry = io.waiters.remove(&fd);
io.epoll_deregister(fd); io.epoll_deregister(fd);
pid entry
}); });
if let Some(pid) = parked_pid { if let Some((pid, epoch)) = parked {
inner.unpark(pid); inner.unpark_at(pid, epoch);
} }
} }
} }
+33 -8
View File
@@ -96,7 +96,10 @@ impl JoinHandle {
Some(cold.outcome.take().expect("Done slot must have outcome")) Some(cold.outcome.take().expect("Done slot must have outcome"))
} }
crate::slot_state::Status::Live => { crate::slot_state::Status::Live => {
cold.waiters.push(me); // begin_wait is lock-free, legal under the cold lock;
// registering under it makes the epoch atomic with
// the check-Done-or-register linearization point.
cold.waiters.push((me, begin_wait()));
None None
} }
} }
@@ -219,11 +222,30 @@ pub fn park_current() {
} }
pub fn unpark(pid: Pid) { pub fn unpark(pid: Pid) {
// The whole protocol lives on the slot's packed word (gen + state in one // The whole protocol lives on the slot's packed word (gen + epoch +
// CAS) — see Slot::unpark_action. No runtime lock unless we enqueue. // state in one CAS) — see slot_state.rs. No runtime lock unless we
// enqueue. WILDCARD form: reserved for terminal wakes (request_stop);
// every registration-based waker must use `unpark_at`.
let _ = try_with_runtime(|inner| inner.unpark(pid)); let _ = try_with_runtime(|inner| inner.unpark(pid));
} }
/// Epoch-matched unpark: wake `pid` only if its current wait is still the
/// one this waker registered for. The form every registration-based waker
/// (channel senders, mutex grants, wait-timers, io completions, joiner
/// wakes) must use — see slot_state.rs for the consuming-wake rules.
pub(crate) fn unpark_at(pid: Pid, epoch: u32) {
let _ = try_with_runtime(|inner| inner.unpark_at(pid, epoch));
}
/// Open a new wait for the CURRENT actor: bump its park-epoch and return
/// it. Call once per wait, before registering `(pid, epoch)` with any
/// waker. Lock-free (one CAS on the own slot word), so it is legal under
/// any lock, including a Channel-class lock.
pub(crate) fn begin_wait() -> u32 {
let me = current_pid().expect("begin_wait() called outside an actor");
with_runtime(|inner| inner.begin_wait(me))
}
/// Request cooperative cancellation of `pid`. /// Request cooperative cancellation of `pid`.
/// ///
/// Sets the actor's stop flag and wakes it so it observes the flag promptly: /// Sets the actor's stop flag and wakes it so it observes the flag promptly:
@@ -281,8 +303,9 @@ impl Drop for NoPreempt {
pub fn sleep(duration: std::time::Duration) { pub fn sleep(duration: std::time::Duration) {
let me = current_pid().expect("sleep() called outside an actor"); let me = current_pid().expect("sleep() called outside an actor");
let _np = NoPreempt::enter(); let _np = NoPreempt::enter();
let epoch = begin_wait();
let deadline = crate::timer::deadline_from_now(duration); let deadline = crate::timer::deadline_from_now(duration);
with_runtime(|inner| inner.timers.lock().unwrap().insert_sleep(deadline, me)); with_runtime(|inner| inner.timers.lock().unwrap().insert_sleep(deadline, me, epoch));
park_current(); park_current();
} }
@@ -290,13 +313,13 @@ pub fn insert_wait_timer(
deadline: std::time::Instant, deadline: std::time::Instant,
pid: Pid, pid: Pid,
target: std::sync::Arc<dyn crate::timer::TimerTarget>, target: std::sync::Arc<dyn crate::timer::TimerTarget>,
wait_seq: u64, epoch: u32,
) { ) {
with_runtime(|inner| { with_runtime(|inner| {
inner.timers.lock().unwrap().insert( inner.timers.lock().unwrap().insert(
deadline, deadline,
pid, pid,
crate::timer::Reason::WaitTimeout { target, wait_seq }, crate::timer::Reason::WaitTimeout { target, epoch },
); );
}); });
} }
@@ -317,9 +340,10 @@ where
}); });
{ {
let _np = NoPreempt::enter(); let _np = NoPreempt::enter();
let epoch = begin_wait();
with_runtime(|inner| { with_runtime(|inner| {
let mut io = inner.io.lock().unwrap(); let mut io = inner.io.lock().unwrap();
io.as_mut().expect("io thread not started").submit(me, work); io.as_mut().expect("io thread not started").submit(me, epoch, work);
}); });
park_current(); park_current();
} }
@@ -351,9 +375,10 @@ pub fn wait_writable(fd: std::os::fd::RawFd) -> std::io::Result<()> {
fn wait_fd(fd: std::os::fd::RawFd, readable: bool, writable: bool) -> std::io::Result<()> { fn wait_fd(fd: std::os::fd::RawFd, readable: bool, writable: bool) -> std::io::Result<()> {
let me = current_pid().expect("wait_*() called outside an actor"); let me = current_pid().expect("wait_*() called outside an actor");
let _np = NoPreempt::enter(); let _np = NoPreempt::enter();
let epoch = begin_wait();
with_runtime(|inner| { with_runtime(|inner| {
let mut io = inner.io.lock().unwrap(); let mut io = inner.io.lock().unwrap();
io.as_mut().expect("io thread not started").epoll_register(fd, me, readable, writable) io.as_mut().expect("io thread not started").epoll_register(fd, me, epoch, readable, writable)
})?; })?;
park_current(); park_current();
Ok(()) Ok(())
+19 -16
View File
@@ -18,9 +18,9 @@
//! No cancellation. When a non-timer wakeup happens (e.g. lock granted //! No cancellation. When a non-timer wakeup happens (e.g. lock granted
//! before timeout), the timer entry is left in the heap. It will be popped //! before timeout), the timer entry is left in the heap. It will be popped
//! eventually and the dispatch will observe "actor is no longer parked / //! eventually and the dispatch will observe "actor is no longer parked /
//! wait_seq is stale" and no-op. Cost is ~32 bytes per stale entry plus a //! the wait's epoch was consumed" and no-op. Cost is ~32 bytes per stale
//! few cycles on pop; acceptable given the upper bound is "one entry per //! entry plus a few cycles on pop; acceptable given the upper bound is "one
//! parked actor". //! entry per parked actor".
//! //!
//! Stale pids (slot reused since the timer was inserted) are filtered on //! Stale pids (slot reused since the timer was inserted) are filtered on
//! pop by the scheduler — same convention as the run queue. //! pop by the scheduler — same convention as the run queue.
@@ -35,18 +35,21 @@ use std::time::{Duration, Instant};
/// ///
/// Held inside `Entry`, dispatched by the scheduler in `pop_due`. /// Held inside `Entry`, dispatched by the scheduler in `pop_due`.
pub enum Reason { pub enum Reason {
/// `loom::sleep(d)`. Unpark `pid` unconditionally (modulo the usual /// `sleep(d)`. Wake `pid` via the epoch-matched unpark: if anything
/// "still parked?" check the scheduler applies). /// else (necessarily a terminal wake) already consumed the wait, the
Sleep, /// entry is stale and no-ops at the CAS.
/// A bounded wait — currently only `Mutex::lock_timeout`. On expiry the Sleep { epoch: u32 },
/// scheduler calls `target.on_timeout(pid, wait_seq)`. The target then /// A bounded wait (`Mutex::lock_timeout`, `Receiver::recv_timeout`,
/// decides whether `pid` was actually still waiting, and if so unparks /// `select_timeout`). On expiry the scheduler calls
/// it with whatever error the wait was bounded for. `wait_seq` lets the /// `target.on_timeout(pid, epoch)`. The target then decides whether
/// target tell apart "this wait" from "a later wait by the same actor /// `pid` was actually still waiting (registration still present under
/// on the same target". /// its lock), and if so takes the registration and unparks via
/// `unpark_at`. The epoch is the slot-word park-epoch — the runtime-wide
/// wait identity — so a stale entry is doubly inert: the registration
/// check misses, and even a racing unpark fails the word's epoch CAS.
WaitTimeout { WaitTimeout {
target: Arc<dyn TimerTarget>, target: Arc<dyn TimerTarget>,
wait_seq: u64, epoch: u32,
}, },
} }
@@ -55,7 +58,7 @@ pub enum Reason {
/// Implementors: do not touch `SchedulerState` other than via the public /// Implementors: do not touch `SchedulerState` other than via the public
/// `unpark` / channel APIs. The scheduler is mid-iteration when this fires. /// `unpark` / channel APIs. The scheduler is mid-iteration when this fires.
pub trait TimerTarget: Send + Sync { pub trait TimerTarget: Send + Sync {
fn on_timeout(&self, pid: Pid, wait_seq: u64); fn on_timeout(&self, pid: Pid, epoch: u32);
} }
pub struct Entry { pub struct Entry {
@@ -104,8 +107,8 @@ impl Timers {
} }
/// Insert a `Sleep` timer. Convenience for the common case. /// Insert a `Sleep` timer. Convenience for the common case.
pub fn insert_sleep(&mut self, deadline: Instant, pid: Pid) { pub fn insert_sleep(&mut self, deadline: Instant, pid: Pid, epoch: u32) {
self.insert(deadline, pid, Reason::Sleep); self.insert(deadline, pid, Reason::Sleep { epoch });
} }
/// Insert an arbitrary timer entry. /// Insert an arbitrary timer entry.
+15 -15
View File
@@ -124,11 +124,11 @@ use smarm::pid::Pid;
use smarm::timer::{Reason, TimerTarget, Timers}; use smarm::timer::{Reason, TimerTarget, Timers};
struct RecordingTarget { struct RecordingTarget {
calls: Mutex<Vec<(Pid, u64)>>, calls: Mutex<Vec<(Pid, u32)>>,
} }
impl TimerTarget for RecordingTarget { impl TimerTarget for RecordingTarget {
fn on_timeout(&self, pid: Pid, seq: u64) { fn on_timeout(&self, pid: Pid, epoch: u32) {
self.calls.lock().unwrap().push((pid, seq)); self.calls.lock().unwrap().push((pid, epoch));
} }
} }
@@ -137,9 +137,9 @@ fn timers_pop_due_returns_entries_in_deadline_order() {
let mut t = Timers::new(); let mut t = Timers::new();
let now = Instant::now(); let now = Instant::now();
// Insert out of order; pop_due should hand them back sorted by deadline. // Insert out of order; pop_due should hand them back sorted by deadline.
t.insert_sleep(now + Duration::from_millis(30), Pid::new(0, 0)); t.insert_sleep(now + Duration::from_millis(30), Pid::new(0, 0), 1);
t.insert_sleep(now + Duration::from_millis(10), Pid::new(1, 0)); t.insert_sleep(now + Duration::from_millis(10), Pid::new(1, 0), 1);
t.insert_sleep(now + Duration::from_millis(20), Pid::new(2, 0)); t.insert_sleep(now + Duration::from_millis(20), Pid::new(2, 0), 1);
// Advance past all of them. // Advance past all of them.
let due = t.pop_due(now + Duration::from_millis(50)); let due = t.pop_due(now + Duration::from_millis(50));
@@ -152,8 +152,8 @@ fn timers_pop_due_returns_entries_in_deadline_order() {
fn timers_only_pop_entries_whose_deadline_has_passed() { fn timers_only_pop_entries_whose_deadline_has_passed() {
let mut t = Timers::new(); let mut t = Timers::new();
let now = Instant::now(); let now = Instant::now();
t.insert_sleep(now + Duration::from_millis(5), Pid::new(0, 0)); t.insert_sleep(now + Duration::from_millis(5), Pid::new(0, 0), 1);
t.insert_sleep(now + Duration::from_millis(100), Pid::new(1, 0)); t.insert_sleep(now + Duration::from_millis(100), Pid::new(1, 0), 1);
let due = t.pop_due(now + Duration::from_millis(20)); let due = t.pop_due(now + Duration::from_millis(20));
assert_eq!(due.len(), 1); assert_eq!(due.len(), 1);
@@ -169,11 +169,11 @@ fn timers_mix_sleep_and_wait_timeout_reasons() {
let target = Arc::new(RecordingTarget { calls: Mutex::new(Vec::new()) }); let target = Arc::new(RecordingTarget { calls: Mutex::new(Vec::new()) });
let now = Instant::now(); let now = Instant::now();
t.insert_sleep(now + Duration::from_millis(5), Pid::new(0, 0)); t.insert_sleep(now + Duration::from_millis(5), Pid::new(0, 0), 1);
t.insert( t.insert(
now + Duration::from_millis(10), now + Duration::from_millis(10),
Pid::new(1, 0), Pid::new(1, 0),
Reason::WaitTimeout { target: target.clone(), wait_seq: 42 }, Reason::WaitTimeout { target: target.clone(), epoch: 42 },
); );
let due = t.pop_due(now + Duration::from_millis(20)); let due = t.pop_due(now + Duration::from_millis(20));
@@ -181,11 +181,11 @@ fn timers_mix_sleep_and_wait_timeout_reasons() {
// Order: Sleep (5ms) first, WaitTimeout (10ms) second. // Order: Sleep (5ms) first, WaitTimeout (10ms) second.
match &due[0].reason { match &due[0].reason {
Reason::Sleep => {} Reason::Sleep { .. } => {}
_ => panic!("first entry should be a Sleep"), _ => panic!("first entry should be a Sleep"),
} }
match &due[1].reason { match &due[1].reason {
Reason::WaitTimeout { wait_seq, .. } => assert_eq!(*wait_seq, 42), Reason::WaitTimeout { epoch, .. } => assert_eq!(*epoch, 42),
_ => panic!("second entry should be a WaitTimeout"), _ => panic!("second entry should be a WaitTimeout"),
} }
} }
@@ -197,9 +197,9 @@ fn same_deadline_entries_pop_in_insertion_order() {
let mut t = Timers::new(); let mut t = Timers::new();
let now = Instant::now(); let now = Instant::now();
let d = now + Duration::from_millis(10); let d = now + Duration::from_millis(10);
t.insert_sleep(d, Pid::new(0, 0)); t.insert_sleep(d, Pid::new(0, 0), 1);
t.insert_sleep(d, Pid::new(1, 0)); t.insert_sleep(d, Pid::new(1, 0), 1);
t.insert_sleep(d, Pid::new(2, 0)); t.insert_sleep(d, Pid::new(2, 0), 1);
let due = t.pop_due(now + Duration::from_millis(20)); let due = t.pop_due(now + Duration::from_millis(20));
let pids: Vec<u32> = due.iter().map(|e| e.pid.index()).collect(); let pids: Vec<u32> = due.iter().map(|e| e.pid.index()).collect();