//! Multi-scheduler runtime: configuration, initialisation, and the shared //! state that all scheduler OS threads operate against. //! //! # Architecture (post slot-table split, ROADMAP_v0.5 phase 2) //! //! ```text //! init(Config) → Runtime (Arc) //! //! RuntimeInner { //! slots: Box<[Slot]> ← FIXED slab, max_actors entries, lock-free lookup //! free: RawMutex> ← vacant slot indices //! run_queue: RunQueue ← compile-time selected (src/run_queue.rs) //! timers: Mutex //! io: Mutex> //! live_actors: AtomicU32 ← spawned-but-not-finalized count (termination) //! stats: Vec ← one per thread, lockless atomics (RFC 000) //! } //! //! Slot { //! word: AtomicU64 ← (gen << 32) | (epoch << 8) | state — THE state machine //! sp: AtomicUsize ← saved stack pointer //! stop_ptr:AtomicPtr<...> ← into the actor's Arc //! closure: AtomicPtr<...> ← first-resume closure, swap-to-take //! cold: RawMutex ← lifecycle collections (waiters/monitors/links/…) //! } //! ``` //! //! # The per-slot state machine //! //! Scheduling state lives in one atomic word per slot packing //! `(generation, park-epoch, state)`, where state is one of: //! //! ```text //! Vacant ─spawn→ Queued ─pop→ Running ─yield→ Queued //! ↑ │ │ //! │ park │ unpark while running //! │ ↓ ↓ //! unpark ←──── Parked RunningNotified ─park→ Queued //! (re-queued immediately) //! Running|RunningNotified ─actor returns→ Done ─reclaim→ Vacant(gen+1) //! ``` //! //! Every transition is a CAS on the packed word, so: //! //! - The generation check is **atomic with the transition** — a stale `Pid` //! can never act on a recycled slot (no ABA, no spurious unparks). //! - The park-epoch (middle 24 bits) is the actor's *wait identity*: opened //! by `begin_wait` before any registration, consumed (bumped) by every //! successful wake. Registration-based wakers carry `(pid, epoch)` and use //! `unpark_at`, so a waker holding a registration from an already-woken //! wait — a `select` loser arm, a satisfied wait's timer — fails the epoch //! check and no-ops instead of faulting a later one-shot park. The only //! wildcard wake is `request_stop`, which is terminal. Full rules in //! slot_state.rs. //! - `RunningNotified` replaces the old `pending_unpark` bool: an unpark that //! races the prep-to-park window is a *state*, resolved by the scheduler's //! park-return CAS, not a flag read under a lock. This also closes a latent //! lost-wakeup in the old Blocking-IO completion path, which set the result //! for a still-Running actor without flagging it. //! - **A pid is in the run queue at most once**: the only pushes are paired //! 1:1 with successful transitions *into* `Queued`, and only the scheduler //! transitions `Queued → Running` (paired 1:1 with pops). //! //! Memory ordering: all word CASes are `AcqRel` (failure `Acquire`), plain //! word stores are `Release`, loads are `Acquire`. The chain that matters: //! the park path stores `sp` (Relaxed) *before* its Release transition; any //! later Acquire transition/load of the word therefore observes that `sp`. //! The run-queue mutex independently provides the same edges today; the //! word's own ordering is what phase 3's lock-free queue will rely on. //! //! # Locks and ordering //! //! - The run queue is its own module (`run_queue.rs`), selected at compile //! time (`rq-mutex` / `rq-mpmc` / `rq-striped`). Queue ops require //! preemption disabled (debug-asserted there); when the mutex variant is in //! play it is the innermost lock — nothing else is acquired under it. //! - Per-slot `cold` locks ([`RawMutex`], non-poisoning, guard enters //! `NoPreempt`) guard the lifecycle collections. **Leaf rule: never hold //! two cold locks at once** — `finalize_actor`'s link cascade and `link()` //! lock peers one at a time (correctness arguments at the call sites). //! Holding a cold lock while pushing to the run queue is permitted. //! - Lock order overall: `io` → (slot `cold` | `free` | `stack_pool`, //! mutually leaf) → run queue (innermost). `timers` is independent (never //! nested with any of the above on either side). //! //! # Termination (counter-based) //! //! The old all-clear scanned the slot table under the big lock. Now: //! exit when `io_out == 0` (read *before* the queue lock, phase-1 ordering) //! and, under the queue lock, the queue is empty and `live_actors == 0`. //! `live_actors` is incremented in `spawn` before the enqueue and decremented //! at the very END of `finalize_actor`, strictly after every wakeup that //! finalize produces has been enqueued. The soundness crux: any enqueue //! targets a live (not-yet-finalized) actor, so `live == 0` implies no wakeup //! can still be in flight; combined with "spawner is itself live", observing //! `(queue empty, live == 0)` under the queue lock means no work can ever //! appear again. //! //! # Timer / IO drain (try-lock, one-winner) //! //! Unchanged from phase 1: one winner per round drains due timers and IO //! completions from their own mutexes; wakeups go through the unpark //! protocol like everyone else's. use crate::actor::{ clear_current_pid, is_actor_done, reset_actor_done, set_current_actor_box, set_current_pid, take_last_outcome, Actor, Outcome, }; use crate::channel::Sender; use crate::io::IoThread; use crate::monitor::{Down, DownReason, MonitorId}; use crate::pid::Pid; use crate::preempt::PREEMPTION_ENABLED; use crate::raw_mutex::RawMutex; use crate::slot_state::{StateWord, Status, Unpark}; use crate::supervisor::Signal; use crate::timer::Timers; use crate::context::{get_actor_sp, set_actor_sp, switch_to_actor}; use std::sync::atomic::{ AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering, }; use std::sync::{Arc, Mutex}; use std::thread; // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- /// Default capacity of the actor slot table. Slots are ~256 bytes, so the /// default costs ~4 MiB, allocated once at `init`. See [`Config::max_actors`]. pub const DEFAULT_MAX_ACTORS: usize = 16_384; /// Runtime configuration. /// /// ``` /// use smarm::runtime::Config; /// /// // Use all available CPUs (default): /// let c = Config::default(); /// /// // Exactly 4 scheduler threads: /// let c = Config::exact(4); /// /// // Between 2 and 8, clamped to available parallelism: /// let c = Config::new(2, 8, None); /// ``` #[derive(Clone, Debug)] pub struct Config { min: usize, max: usize, exact: Option, alloc_interval: u32, timeslice_cycles: u64, stack_pool_cap: usize, max_actors: usize, } impl Config { /// Exact thread count; takes precedence over min/max. pub fn exact(n: usize) -> Self { assert!(n >= 1, "scheduler thread count must be ≥ 1"); Self { min: n, max: n, exact: Some(n), alloc_interval: crate::preempt::DEFAULT_ALLOC_INTERVAL, timeslice_cycles: crate::preempt::DEFAULT_TIMESLICE_CYCLES, stack_pool_cap: n * 4, max_actors: DEFAULT_MAX_ACTORS, } } /// Bounded range. Thread count = clamp(available_parallelism, min, max). pub fn new(min: usize, max: usize, exact: Option) -> Self { assert!(min >= 1, "min must be ≥ 1"); assert!(max >= min, "max must be ≥ min"); if let Some(e) = exact { assert!(e >= 1, "exact must be ≥ 1"); } Self { min, max, exact, alloc_interval: crate::preempt::DEFAULT_ALLOC_INTERVAL, timeslice_cycles: crate::preempt::DEFAULT_TIMESLICE_CYCLES, stack_pool_cap: max * 4, max_actors: DEFAULT_MAX_ACTORS, } } /// How many allocations (or `smarm::check!()` calls) between RDTSC checks. /// Lower = more responsive preemption, higher = less overhead. /// Default: 128. pub fn alloc_interval(mut self, n: u32) -> Self { assert!(n >= 1, "alloc_interval must be ≥ 1"); self.alloc_interval = n; self } /// How many TSC cycles constitute one timeslice. /// Default: 300_000 (≈ 100µs on a 3 GHz CPU). pub fn timeslice_cycles(mut self, n: u64) -> Self { assert!(n >= 1, "timeslice_cycles must be ≥ 1"); self.timeslice_cycles = n; self } /// Maximum number of stacks kept in the pool for reuse across spawns. /// A larger cap reduces `mmap`/`munmap` syscalls at the cost of idle memory. /// Default: `thread_count * 4`. pub fn stack_pool_cap(mut self, n: usize) -> Self { self.stack_pool_cap = n; self } /// Capacity of the actor slot table — the maximum number of /// **simultaneously live** actors (total spawned over a run is unbounded; /// slots are recycled). The table is a fixed slab allocated once at /// `init`: slots never move, which is what makes lock-free slot lookup /// sound. Exhausting it is a loud panic naming this knob. /// Default: [`DEFAULT_MAX_ACTORS`] (16_384, ~4 MiB). pub fn max_actors(mut self, n: usize) -> Self { assert!(n >= 1, "max_actors must be ≥ 1"); assert!( n < u32::MAX as usize, "max_actors must fit a u32 slot index (ROOT_PID reserves u32::MAX)" ); self.max_actors = n; self } /// The number of scheduler threads this config resolves to. pub fn resolved_thread_count(&self) -> usize { if let Some(e) = self.exact { return e; } let avail = thread::available_parallelism() .map(|n| n.get()) .unwrap_or(1); avail.clamp(self.min, self.max) } } impl Default for Config { fn default() -> Self { let avail = thread::available_parallelism() .map(|n| n.get()) .unwrap_or(1); Self { min: 1, max: avail, exact: None, alloc_interval: crate::preempt::DEFAULT_ALLOC_INTERVAL, timeslice_cycles: crate::preempt::DEFAULT_TIMESLICE_CYCLES, stack_pool_cap: avail * 4, max_actors: DEFAULT_MAX_ACTORS, } } } // --------------------------------------------------------------------------- // Per-thread stats (RFC 000 Layer 1 primitives) // --------------------------------------------------------------------------- /// Lockless per-scheduler-thread counters. Written only by the owning thread; /// readable from any thread (introspection actor, tests). pub struct SchedulerStats { /// PID index of the actor currently on-CPU, or `u32::MAX` when idle. pub current_pid_index: AtomicU32, /// Snapshot of run queue length maintained on every push/pop. pub run_queue_len: AtomicU64, } impl SchedulerStats { fn new() -> Self { Self { current_pid_index: AtomicU32::new(u32::MAX), run_queue_len: AtomicU64::new(0), } } } // --------------------------------------------------------------------------- // Runtime stats snapshot (for tests / introspection) // --------------------------------------------------------------------------- pub struct RuntimeStats { pub(crate) inner: Arc, } impl RuntimeStats { /// Sum of run queue lengths across all scheduler threads. pub fn total_run_queue_len(&self) -> u64 { self.inner.stats.iter() .map(|s| s.run_queue_len.load(Ordering::Relaxed)) .sum() } /// Number of scheduler threads. pub fn scheduler_count(&self) -> usize { self.inner.stats.len() } /// Actors currently parked on IO. pub fn io_parked_count(&self) -> u32 { self.inner.io_parked.load(Ordering::Relaxed) } /// Actors currently sleeping on a timer. pub fn sleeping_count(&self) -> u32 { self.inner.sleeping.load(Ordering::Relaxed) } } // --------------------------------------------------------------------------- // Slot — packed state word + hot atomics + cold lifecycle data // --------------------------------------------------------------------------- pub(crate) const ACTOR_STACK_SIZE: usize = 64 * 1024; pub(crate) type Closure = Box; /// Lifecycle data, mutated only under the slot's cold [`RawMutex`]. Everything /// here is touched O(1) times per actor lifetime (spawn / join / monitor / /// link / finalize), never on the yield/park/unpark hot path. pub(crate) struct SlotCold { pub(crate) actor: Option, /// Parked joiners as `(pid, park-epoch)`; finalize wakes each via the /// epoch-matched unpark. pub(crate) waiters: Vec<(Pid, u32)>, pub(crate) outcome: Option, pub(crate) supervisor_channel: Option>, /// Watchers registered via `monitor()`, each tagged with its /// `MonitorId` so `demonitor` can remove exactly one. Each receives one /// `Down` when this actor terminates (drained in `finalize_actor`). /// Distinct from `supervisor_channel`, which is the parent's single funnel. pub(crate) monitors: Vec<(MonitorId, Sender)>, /// Bidirectional links (roadmap #3). Each entry is a peer whose abnormal /// death propagates to this actor (and vice versa). Entries may be /// momentarily or persistently stale (peer already dead) — every walk /// re-verifies the peer's word, so stale entries are benign no-ops. pub(crate) links: Vec, pub(crate) outstanding_handles: u32, pub(crate) pending_io_result: Option, } /// One actor slot. Hot scheduling state is atomic; cold lifecycle state is /// behind `cold`. Slots live in a fixed slab and never move. /// /// `align(128)` keeps two adjacent slots' hot words off each other's /// cache-line pair (x86 prefetches lines in pairs), avoiding false sharing /// between unrelated actors. #[repr(align(128))] pub(crate) struct Slot { /// `(generation << 32) | state` — the state machine, factored into /// `slot_state.rs` (loom-modeled there; every transition self-asserts). word: StateWord, /// Saved stack pointer. Written by the owning scheduler thread before the /// Release transition out of Running; read after the Acquire transition /// Queued→Running. Relaxed is sufficient — ordering rides on `word`. sp: AtomicUsize, /// Pointer into the actor's `Arc` stop flag. Set at spawn, /// nulled at finalize. The box outlives every read: it is only ever read /// on the resume path while the actor cannot be finalized (it is on-CPU). stop_ptr: AtomicPtr, /// First-resume closure, double-boxed so it fits an `AtomicPtr` /// (`Box` is a thin pointer). Swap-to-take; null when absent. closure: AtomicPtr, /// Cold lifecycle data. See [`SlotCold`]. pub(crate) cold: RawMutex, } impl Slot { fn vacant() -> Self { Self { word: StateWord::new(), sp: AtomicUsize::new(0), stop_ptr: AtomicPtr::new(std::ptr::null_mut()), closure: AtomicPtr::new(std::ptr::null_mut()), cold: RawMutex::new(SlotCold { actor: None, waiters: Vec::new(), outcome: None, supervisor_channel: None, monitors: Vec::new(), links: Vec::new(), outstanding_handles: 0, pending_io_result: None, }), } } /// Current generation (of whatever occupies the slot — pair with a /// status check or a CAS before acting on it). #[inline] pub(crate) fn generation(&self) -> u32 { self.word.generation() } /// A pid's-eye snapshot of the slot. Cold paths re-read this under the /// cold lock (generation can't change while it is held). #[inline] pub(crate) fn status_for(&self, pid: Pid) -> Status { self.word.status_for(pid.generation()) } /// Does the slot currently hold the actor `pid` names, in a non-terminal /// state? (Snapshot — callers that mutate must re-verify under `cold` or /// CAS on the word.) #[inline] pub(crate) fn is_live_for(&self, pid: Pid) -> bool { self.status_for(pid) == Status::Live } fn store_closure(&self, c: Closure) { let raw = Box::into_raw(Box::new(c)); let prev = self.closure.swap(raw, Ordering::Release); debug_assert!(prev.is_null(), "slot already had a pending closure"); } fn take_closure(&self) -> Option { let raw = self.closure.swap(std::ptr::null_mut(), Ordering::Acquire); if raw.is_null() { None } else { // SAFETY: non-null values in `closure` are exclusively // `Box::into_raw(Box)` from `store_closure`, and the // swap above made us the unique owner. Some(*unsafe { Box::from_raw(raw) }) } } } // --------------------------------------------------------------------------- // RuntimeInner — the shared core behind an Arc // --------------------------------------------------------------------------- pub(crate) struct RuntimeInner { /// The run queue, compile-time selected (see `run_queue.rs` for the /// contract: ops require preemption disabled, push is infallible, /// pop-None is a snapshot). pub(crate) run_queue: crate::run_queue::RunQueue, /// The fixed actor slot table. Allocated once; slots never move. pub(crate) slots: Box<[Slot]>, /// Vacant slot indices. RawMutex leaf; never held with a cold lock. pub(crate) free: RawMutex>, /// Spawned-but-not-finalized actor count; the termination criterion. /// Incremented in `spawn` before the enqueue; decremented at the very end /// of `finalize_actor`, after every wakeup finalize produces. pub(crate) live_actors: AtomicU32, /// Timer heap. Independent lock: never nested with any other. pub(crate) timers: Mutex, /// IO subsystem. `None` between runs. Lock order: io before everything. pub(crate) io: Mutex>, /// Monotonic `MonitorId` source. Never reused. pub(crate) next_monitor_id: AtomicU64, /// Try-lock: exactly one scheduler thread drains timers/IO per iteration. drain_lock: Mutex<()>, /// Per-thread stats, indexed by scheduler thread slot (0..N). pub(crate) stats: Vec, /// Global counters for RFC 000 primitives. pub(crate) io_parked: AtomicU32, pub(crate) sleeping: AtomicU32, /// Preemption knobs, written into each scheduler thread's locals on startup. pub(crate) alloc_interval: u32, pub(crate) timeslice_cycles: u64, /// The name <-> pid registry (bidirectional). RawMutex Leaf: never held /// with any other lock; liveness checks under it read only the atomic /// slot word. pub(crate) registry: RawMutex, /// Recycled stacks waiting to be reused by the next spawn. pub(crate) stack_pool: RawMutex>, /// Maximum number of stacks to retain in the pool. pub(crate) stack_pool_cap: usize, } impl RuntimeInner { fn new( thread_count: usize, alloc_interval: u32, timeslice_cycles: u64, stack_pool_cap: usize, max_actors: usize, ) -> Arc { let stats = (0..thread_count).map(|_| SchedulerStats::new()).collect(); let slots: Box<[Slot]> = (0..max_actors).map(|_| Slot::vacant()).collect(); // Low indices on top of the stack so early spawns get low pids. let free: Vec = (0..max_actors as u32).rev().collect(); Arc::new(Self { run_queue: crate::run_queue::RunQueue::new(thread_count, max_actors), slots, free: RawMutex::new(free), live_actors: AtomicU32::new(0), timers: Mutex::new(Timers::new()), io: Mutex::new(None), next_monitor_id: AtomicU64::new(0), drain_lock: Mutex::new(()), stats, io_parked: AtomicU32::new(0), sleeping: AtomicU32::new(0), alloc_interval, timeslice_cycles, registry: RawMutex::new(crate::registry::Registry::new()), stack_pool: RawMutex::new(Vec::new()), stack_pool_cap, }) } /// Slot lookup by index only — bounds-checked, NOT generation-checked. /// `ROOT_PID` (index `u32::MAX`) is out of bounds by construction and /// resolves to `None`. Callers verify the generation atomically: either /// inside a CAS on the word, or by re-reading the word under the cold lock. #[inline] pub(crate) fn slot_at(&self, pid: Pid) -> Option<&Slot> { self.slots.get(pid.index() as usize) } /// Push to the run queue. Callers must have just transitioned the pid /// into `Queued` (spawn's publish, the unpark protocol, or the /// scheduler's yield/notified-park return paths). pub(crate) fn enqueue(&self, pid: Pid) { // Every push pairs 1:1 with a transition INTO Queued, and nothing can // move the word off Queued until this very entry is popped — so the // word must read EXACTLY (gen, Queued) here. This is the at-most-once- // enqueued invariant the bounded rings' capacity proof leans on. debug_assert!( self.slot_at(pid).map(|s| s.word.load()).is_some_and(|w| { crate::slot_state::word_gen(w) == pid.generation() && crate::slot_state::word_state(w) == crate::slot_state::ST_QUEUED }), "enqueue of a pid not in (gen, Queued)" ); self.run_queue.push(pid); crate::te!(crate::trace::Event::Enqueue(pid)); } /// Make `pid` runnable if it is parked; coalesce or defer otherwise. /// The runtime-internal core of `scheduler::unpark`. WILDCARD wake: /// consumes the epoch but does not check it — reserved for terminal /// wakes (`request_stop`); see slot_state.rs. pub(crate) fn unpark(&self, pid: Pid) { self.unpark_inner(pid, None); } /// Epoch-matched wake: lands only if `pid`'s current wait is still the /// one the waker registered for. The form every registration-based waker /// (channel senders, mutex grants, wait-timers, …) must use. pub(crate) fn unpark_at(&self, pid: Pid, epoch: u32) { self.unpark_inner(pid, Some(epoch)); } /// Open a new wait for `pid` (the calling actor itself): bump its /// park-epoch and return it. See slot_state.rs for the rules. #[must_use] pub(crate) fn begin_wait(&self, pid: Pid) -> u32 { let slot = self.slot_at(pid).expect("begin_wait: own slot vanished"); slot.word.begin_wait(pid.generation()) } /// Retire the calling actor's current wait without parking on it: bump /// the epoch (invalidating every in-flight registration-based wake), /// then eat a notification that already landed. The caller MUST /// re-check its stop flag afterwards — see `StateWord::clear_notify`. pub(crate) fn retire_wait(&self, pid: Pid) { let slot = self.slot_at(pid).expect("retire_wait: own slot vanished"); let _ = slot.word.begin_wait(pid.generation()); slot.word.clear_notify(pid.generation()); } fn unpark_inner(&self, pid: Pid, want: Option) { if let Some(slot) = self.slot_at(pid) { match slot.word.unpark(pid.generation(), want) { Unpark::Enqueue => { crate::te!(crate::trace::Event::UnparkDirect(pid)); self.enqueue(pid); } Unpark::Notified => { crate::te!(crate::trace::Event::UnparkDeferred(pid)); } Unpark::Noop => {} } } } /// Allocate the next process-unique `MonitorId`. Lock-free; monitors are a /// cold path but there is no reason to serialize id minting under any lock. pub(crate) fn alloc_monitor_id(&self) -> MonitorId { MonitorId(self.next_monitor_id.fetch_add(1, Ordering::Relaxed) + 1) } /// Pop a vacant slot index, or die loudly. The fixed slab is a deliberate /// v0.5 simplification (ROADMAP: "Deferred"); the panic names the fix. pub(crate) fn allocate_slot(&self) -> u32 { match self.free.lock().pop() { Some(idx) => idx, None => panic!( "smarm: actor slot table exhausted — {} actors are live \ simultaneously, which is the configured maximum. \ Fix: raise the cap at runtime init, e.g. \ `smarm::init(Config::default().max_actors({}))`. \ (Slots are ~256 bytes each; the table is allocated up-front.)", self.slots.len(), self.slots.len() * 2 ), } } } // --------------------------------------------------------------------------- // Runtime — the public handle // --------------------------------------------------------------------------- pub struct Runtime { inner: Arc, thread_count: usize, } /// Initialise the runtime with the given config. Returns a reusable handle. pub fn init(config: Config) -> Runtime { let n = config.resolved_thread_count(); Runtime { inner: RuntimeInner::new( n, config.alloc_interval, config.timeslice_cycles, config.stack_pool_cap, config.max_actors, ), thread_count: n, } } impl Runtime { /// Run `f` as the initial actor, block until all actors finish. /// Can be called multiple times sequentially on the same `Runtime`. pub fn run(&self, f: impl FnOnce() + Send + 'static) { // Install smarm's panic hook on first call. The default Rust hook is // not reentrant — concurrent actor panics can trigger a double-panic // abort when the backtrace printer takes an internal lock that is // already held. smarm catches every actor panic via `catch_unwind` in // the trampoline, so panics never need to reach the hook for runtime // correctness; the hook fires only as a side-effect of unwinding before // `catch_unwind` catches it. // // We install once and leave it installed: the previous hook is chained // so that panics outside actor context (e.g. in the test harness // itself) are still reported normally. static HOOK_INSTALLED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); HOOK_INSTALLED.get_or_init(|| { let prev = std::panic::take_hook(); std::panic::set_hook(Box::new(move |info| { // If we are currently executing inside an actor trampoline the // panic will be caught by `catch_unwind` momentarily. Suppress // the hook output to avoid interleaved noise and reentrancy. // Outside actor context, delegate to the previous hook so that // genuine runtime panics are still reported. if crate::actor::current_pid().is_some() { // Inside an actor — catch_unwind handles it; stay silent. } else { prev(info); } })); }); // Open the trace store for this run (no-op without smarm-trace). #[cfg(feature = "smarm-trace")] crate::trace::open(); // Re-initialise shared state for this run. assert_eq!( self.inner.run_queue.len(), 0, "run() called while previous run still active" ); debug_assert_eq!( self.inner.live_actors.load(Ordering::Acquire), 0, "run() called while previous run still active" ); *self.inner.io.lock().unwrap() = Some(IoThread::start().expect("failed to start IO thread")); // Spawn the initial actor through the public spawn path (which // requires a running runtime in the thread-local). RUNTIME.with(|r| *r.borrow_mut() = Some(self.inner.clone())); let initial_handle = crate::scheduler::spawn(f); // Launch N-1 extra scheduler threads. The calling thread is thread 0. let mut os_threads = Vec::new(); for slot in 1..self.thread_count { let inner = self.inner.clone(); let t = thread::spawn(move || { RUNTIME.with(|r| *r.borrow_mut() = Some(inner.clone())); SCHED_SLOT.with(|s| s.set(slot)); schedule_loop(&inner, slot); RUNTIME.with(|r| *r.borrow_mut() = None); }); os_threads.push(t); } // Thread 0 runs the loop on the calling thread. SCHED_SLOT.with(|s| s.set(0)); schedule_loop(&self.inner, 0); // Wait for all other scheduler threads. for t in os_threads { let _ = t.join(); } // Drop initial handle (decrements outstanding_handles count). drop(initial_handle); // Tear down IO and clean up for the next run() call. drop(self.inner.io.lock().unwrap().take()); // joins IO threads self.inner.timers.lock().unwrap().clear(); self.inner.next_monitor_id.store(0, Ordering::Relaxed); // Every slot must have come back: any leak here is a runtime bug // (a JoinHandle held across run() is decremented just above). debug_assert_eq!( self.inner.free.lock().len(), self.inner.slots.len(), "slot leak across run()" ); // Reset per-thread stats. for stat in &self.inner.stats { stat.current_pid_index.store(u32::MAX, Ordering::Relaxed); stat.run_queue_len.store(0, Ordering::Relaxed); } self.inner.io_parked.store(0, Ordering::Relaxed); self.inner.sleeping.store(0, Ordering::Relaxed); RUNTIME.with(|r| *r.borrow_mut() = None); // Flush trace to disk (no-op without smarm-trace). #[cfg(feature = "smarm-trace")] crate::trace::flush(); } /// Snapshot of runtime statistics for introspection / tests. pub fn stats(&self) -> RuntimeStats { RuntimeStats { inner: self.inner.clone() } } } // --------------------------------------------------------------------------- // Thread-locals // --------------------------------------------------------------------------- use std::cell::{Cell, RefCell}; thread_local! { /// The RuntimeInner for the current run(). Set by run() on the calling /// thread and by each spawned scheduler thread. pub(crate) static RUNTIME: RefCell>> = const { RefCell::new(None) }; /// This scheduler thread's index into RuntimeInner::stats. static SCHED_SLOT: Cell = const { Cell::new(0) }; /// What the actor wants when it yields back to the scheduler. static YIELD_INTENT: Cell = const { Cell::new(YieldIntent::Yield) }; } #[derive(Copy, Clone)] pub(crate) enum YieldIntent { Yield, Park } pub(crate) fn set_yield_intent(i: YieldIntent) { YIELD_INTENT.with(|c| c.set(i)); } // --------------------------------------------------------------------------- // Sentinel root PID // --------------------------------------------------------------------------- /// Index `u32::MAX` is out of bounds for any slab (Config asserts /// `max_actors < u32::MAX`), so every slot lookup on ROOT_PID resolves to /// `None` — the root "actor" silently absorbs supervisor signals. pub const ROOT_PID: Pid = Pid::new(u32::MAX, u32::MAX); // --------------------------------------------------------------------------- // Spawn-side slot installation // --------------------------------------------------------------------------- /// Install a freshly spawned actor into the slot `idx` (which must have come /// from `allocate_slot`) and publish it as Queued. Returns the new `Pid`. /// Called by `scheduler::spawn_under`; lives here next to its inverse /// (`reclaim_slot`) so the lifecycle is in one file. pub(crate) fn install_actor( inner: &RuntimeInner, idx: u32, sp: usize, stack: crate::stack::Stack, supervisor: Pid, closure: Closure, ) -> Pid { let slot = &inner.slots[idx as usize]; let gen = slot.generation(); // stable: we own the vacant slot via the free list let pid = Pid::new(idx, gen); let stop = Arc::new(AtomicBool::new(false)); slot.stop_ptr.store(Arc::as_ptr(&stop) as *mut _, Ordering::Release); { let mut cold = slot.cold.lock(); debug_assert!(cold.actor.is_none(), "install over live actor"); debug_assert!(cold.waiters.is_empty() && cold.monitors.is_empty() && cold.links.is_empty()); cold.actor = Some(Actor { pid, stack, supervisor, stop, trap: None }); cold.outstanding_handles = 1; cold.outcome = None; cold.pending_io_result = None; } slot.sp.store(sp, Ordering::Relaxed); slot.store_closure(closure); inner.live_actors.fetch_add(1, Ordering::Relaxed); // Publish: only now can pops, unparks, or stops find the actor. The // Release store orders everything above before any Acquire reader. slot.word.publish_queued(gen); inner.enqueue(pid); crate::te!(crate::trace::Event::Spawn { parent: supervisor, child: pid }); pid } // --------------------------------------------------------------------------- // Slot reclamation // --------------------------------------------------------------------------- /// Reclaim `pid`'s slot if (still) eligible: generation matches, state is /// Done, and no handles are outstanding. Safe to call from racing sites /// (finalize tail vs. JoinHandle drop): the first caller bumps the /// generation under the cold lock, the loser sees the mismatch and no-ops. /// /// Channel senders extracted from the slot are dropped *after* the cold lock /// is released — a last-sender drop can unpark a receiver, which takes the /// run-queue mutex; legal under a cold lock, but pointless to nest. pub(crate) fn reclaim_slot(inner: &RuntimeInner, pid: Pid) { let Some(slot) = inner.slot_at(pid) else { return }; let dropped_outside; { let mut cold = slot.cold.lock(); if slot.status_for(pid) != Status::Done || cold.outstanding_handles != 0 { return; // already reclaimed, or not yet eligible } debug_assert!(cold.actor.is_none(), "reclaiming a slot that still owns an actor"); dropped_outside = ( cold.outcome.take(), cold.supervisor_channel.take(), cold.pending_io_result.take(), slot.take_closure(), // an actor stopped before first resume ); cold.waiters.clear(); cold.monitors.clear(); cold.links.clear(); slot.stop_ptr.store(std::ptr::null_mut(), Ordering::Release); // The generation bump IS the reclaim: every stale pid is dead from // this store onwards (unpark protocol, pops, cold-path re-verifies). slot.word.reclaim(pid.generation()); } drop(dropped_outside); inner.free.lock().push(pid.index()); } // --------------------------------------------------------------------------- // finalize_actor // --------------------------------------------------------------------------- fn finalize_actor(inner: &Arc, pid: Pid, outcome: Outcome) { let (joiner_outcome, sup_signal, down_reason) = match outcome { Outcome::Exit => (Outcome::Exit, Signal::Exit(pid), DownReason::Exit), Outcome::Panic(payload) => ( Outcome::Panic(payload), Signal::Panic(pid, Box::new(()) as Box), DownReason::Panic, ), // Cooperative cancellation: kept distinct from a normal Exit so a // supervisor's await logic (roadmap #2) can tell "I stopped it" apart // from "it finished on its own". Outcome::Stopped => (Outcome::Stopped, Signal::Stopped(pid), DownReason::Stopped), }; let slot = inner.slot_at(pid).expect("finalize_actor: pid out of range"); let (waiters, monitors, links, actor) = { let mut cold = slot.cold.lock(); let actor = cold.actor.take().expect("finalize_actor: actor vanished"); cold.outcome = Some(joiner_outcome); slot.stop_ptr.store(std::ptr::null_mut(), Ordering::Release); // Done is published under the cold lock, so join's // check-Done-or-register-waiter (also under it) can never miss: it // either sees Done and takes the outcome, or its waiter registration // happens before our take() below and is woken further down. // (set_done self-asserts the Running|Notified precondition + gen.) slot.word.set_done(pid.generation()); ( std::mem::take(&mut cold.waiters), std::mem::take(&mut cold.monitors), std::mem::take(&mut cold.links), actor, ) }; // Recycle the stack outside the cold lock; drop the rest of the Actor // (the trap sender can unpark its receiver — keep that outside too). let supervisor_pid = actor.supervisor; let Actor { stack, .. } = actor; { let mut pool = inner.stack_pool.lock(); if pool.len() < inner.stack_pool_cap { pool.push(stack); } // else: drop here → munmap, same as before } // Deliver to supervisor. ROOT_PID resolves to no slot → silently absorbed. let sender = inner.slot_at(supervisor_pid).and_then(|sup| { let cold = sup.cold.lock(); if sup.generation() == supervisor_pid.generation() { cold.supervisor_channel.clone() } else { None } }); if let Some(sender) = sender { let _ = sender.send(sup_signal); } // Notify monitors. Sent outside any slot lock: `send` may unpark a parked // receiver, which takes the run-queue mutex. for (_, m) in monitors { let _ = m.send(Down { pid, reason: down_reason }); } // Walk linked peers ONE AT A TIME (cold locks are leaves). For every // peer: remove the back-link to this (dying) actor; on abnormal death, // also fetch its trap sender and deliver after unlocking. // // Acyclicity: the back-link removal happens under the peer's cold lock // *before* any stop is delivered to it, so when the peer later dies its // own cascade no longer contains us. Two peers finalizing concurrently // each find the other already Done (set above, before any cascade) and // skip — no ping-pong, no deadlock (never two cold locks held). let abnormal = matches!(down_reason, DownReason::Panic | DownReason::Stopped); for peer in links { let trap = match inner.slot_at(peer) { Some(ps) => { let mut cold = ps.cold.lock(); if ps.status_for(peer) == Status::Live { cold.links.retain(|p| *p != pid); if abnormal { Some(cold.actor.as_ref().and_then(|a| a.trap.clone())) } else { None // normal exit never propagates } } else { None // peer already gone; nothing to do } } None => None, }; match trap { Some(Some(tx)) => { let _ = tx.send(crate::link::ExitSignal { from: pid, reason: down_reason }); } Some(None) => crate::scheduler::request_stop(peer), None => {} } } // Unpark joiners (epoch-matched: each registered under the cold lock). for (joiner, epoch) in waiters { inner.unpark_at(joiner, epoch); } // Reclaim if no outstanding handles (re-verified inside). reclaim_slot(inner, pid); // The decrement is LAST: every wakeup this finalize produced (joiners, // monitor/trap sends, stop cascades) is enqueued before `live_actors` // can be observed at its decremented value. See the termination note in // the module docs. let prev = inner.live_actors.fetch_sub(1, Ordering::Release); debug_assert!(prev >= 1, "live_actors underflow — double finalize"); } // --------------------------------------------------------------------------- // schedule_loop — runs on each scheduler OS thread // --------------------------------------------------------------------------- fn schedule_loop(inner: &Arc, slot_idx: usize) { crate::preempt::configure_preempt(inner.alloc_interval, inner.timeslice_cycles); let stats = &inner.stats[slot_idx]; loop { // ---------------------------------------------------------------- // 1. Try to win the drain lock (timers + IO). One winner per round; // losers skip immediately and proceed to step 2. // ---------------------------------------------------------------- if let Ok(_drain_guard) = inner.drain_lock.try_lock() { // Timers and IO live behind their own mutexes (phase 1), so the // pure-yield / pure-compute hot path never contends a global lock // just to discover there is nothing to drain. The clock is read // only when the timer heap is non-empty. let due = { let mut t = inner.timers.lock().unwrap(); if t.is_empty() { Vec::new() } else { t.pop_due(std::time::Instant::now()) } }; let completions = inner.io.lock().unwrap() .as_mut() .map(|io| io.drain_completions()) .unwrap_or_default(); for entry in due { match entry.reason { // A sleep expiry is just an unpark: the protocol handles // every interleaving — Parked (re-queue), Running (the // actor is between `timers.insert_sleep` and // `park_current`; RunningNotified makes the upcoming park // re-queue), or gone (no-op). crate::timer::Reason::Sleep { epoch } => { inner.unpark_at(entry.pid, epoch) } crate::timer::Reason::WaitTimeout { target, epoch } => { // The callback may call unpark_at itself. target.on_timeout(entry.pid, epoch); } } } for completion in completions { match completion { crate::io::Completion::Blocking { pid, epoch, result } => { if let Some(io) = inner.io.lock().unwrap().as_mut() { io.outstanding = io.outstanding.saturating_sub(1); } // Stash the result under the cold lock, then unpark. // The protocol also covers the submit→park window // (RunningNotified), which the old code missed for // Blocking completions — a latent lost wakeup. if let Some(slot) = inner.slot_at(pid) { { let mut cold = slot.cold.lock(); if slot.generation() == pid.generation() { cold.pending_io_result = Some(result); } else { // Actor died (stopped) with the op in // flight; discard the result. } } inner.unpark_at(pid, epoch); } } crate::io::Completion::FdReady { fd, events: _ } => { // Resolve the parked pid under the io lock, then wake // through the protocol. Lock order: io before all. let parked = inner.io.lock().unwrap().as_mut().and_then(|io| { let entry = io.waiters.remove(&fd); io.epoll_deregister(fd); entry }); if let Some((pid, epoch)) = parked { inner.unpark_at(pid, epoch); } } } } } // drain_guard drops here // ---------------------------------------------------------------- // 2. Pop a runnable pid. The queue mutex covers ONLY the pop; the // slot's own atomics carry everything needed to resume. // ---------------------------------------------------------------- enum Pop { Got(Pid), Idle { io_outstanding: u32, wake_fd: Option }, AllDone, } // Read IO liveness BEFORE the queue lock (phase-1 ordering: a // completion resurrects an actor only via the drain path, whose // enqueue would be visible under the queue lock we take next). let (io_out, io_fd) = match inner.io.lock().unwrap().as_ref() { Some(io) => (io.outstanding + io.waiters.len() as u32, Some(io.wake_fd())), None => (0, None), }; stats.run_queue_len.store(inner.run_queue.len(), Ordering::Relaxed); let pop = match inner.run_queue.pop() { Some(pid) => Pop::Got(pid), None => { // Termination does not lean on pop-None being a fence (with // the ring queues it is only a snapshot). The argument is // counter-first: every queue entry's target stays `Queued` — // hence un-finalized, hence counted live — until that very // entry is popped. So `live == 0` (Acquire, pairing with // finalize's Release decrement, which strictly follows all // wakeup enqueues) by itself implies no entry is in, or can // ever again enter, the queue: enqueues only target live // actors, and a spawner is itself live. The pop-None above // is then just the cheap fast-path filter; io_out was read // before it per the phase-1 ordering. `live == 0` is also // final — no spawn can resurrect the count — so every // scheduler thread independently reaches this same verdict. let live = inner.live_actors.load(Ordering::Acquire); if live == 0 && io_out == 0 { Pop::AllDone } else { Pop::Idle { io_outstanding: io_out, wake_fd: io_fd } } } }; let pid = match pop { Pop::Got(pid) => pid, Pop::AllDone => { // Remaining timer entries are orphaned (no live actor can be // woken by them — e.g. a sleeper cancelled out of its sleep); // they must not keep the runtime alive. Drop them on the way out. inner.timers.lock().unwrap().clear(); return; } Pop::Idle { io_outstanding, wake_fd } => { // Something is still in flight. Sleep on the appropriate // source to avoid hammering the queue mutex; retry on wake. let next_deadline = inner.timers.lock().unwrap().peek_deadline(); match (next_deadline, wake_fd) { (Some(deadline), fd_opt) => { let now = std::time::Instant::now(); if deadline > now { let timeout = deadline - now; match fd_opt { Some(fd) => { crate::io::poll_wake(fd, Some(timeout)); crate::io::drain_wake_pipe(fd); } None => thread::sleep(timeout), } } } (None, Some(fd)) if io_outstanding > 0 => { crate::io::poll_wake(fd, None); crate::io::drain_wake_pipe(fd); } _ => { thread::sleep(std::time::Duration::from_micros(100)); } } continue; } }; // ---------------------------------------------------------------- // 3. Claim and resume the actor: CAS Queued → Running. A failure // means the pid is stale (slot recycled — generation mismatch); // by the at-most-once-enqueued invariant nothing else can have // changed the state of a queued actor. // ---------------------------------------------------------------- let slot = match inner.slot_at(pid) { Some(s) => s, None => continue, // can't happen for real pids; defensive }; if !slot.word.try_claim(pid.generation()) { continue; // stale pid: retry immediately (never the idle path) } crate::te!(crate::trace::Event::Dequeue(pid)); let sp = slot.sp.load(Ordering::Relaxed); let stop_flag = slot.stop_ptr.load(Ordering::Relaxed); // First resume: move the closure into the trampoline's thread-local. if let Some(b) = slot.take_closure() { set_current_actor_box(b); } // Update per-thread stats: record who's on-CPU. stats.current_pid_index.store(pid.index(), Ordering::Relaxed); set_actor_sp(sp); set_current_pid(pid); crate::preempt::set_current_stop(stop_flag); reset_actor_done(); YIELD_INTENT.with(|c| c.set(YieldIntent::Yield)); crate::preempt::reset_timeslice(); PREEMPTION_ENABLED.with(|c| c.set(true)); crate::te!(crate::trace::Event::Resume(pid)); unsafe { switch_to_actor() }; PREEMPTION_ENABLED.with(|c| c.set(false)); stats.current_pid_index.store(u32::MAX, Ordering::Relaxed); clear_current_pid(); crate::preempt::clear_current_stop(); let intent = YIELD_INTENT.with(|c| c.get()); slot.sp.store(get_actor_sp(), Ordering::Relaxed); if is_actor_done() { crate::te!(crate::trace::Event::Done(pid)); let outcome = take_last_outcome().unwrap_or(Outcome::Exit); finalize_actor(inner, pid, outcome); } else { let gen = pid.generation(); match intent { YieldIntent::Yield => { // Running OR RunningNotified → Queued; a notification // arriving mid-run coalesces into the re-queue. crate::te!(crate::trace::Event::Yield(pid)); slot.word.yield_return(gen); inner.enqueue(pid); } YieldIntent::Park => { if slot.word.park_return(gen) { crate::te!(crate::trace::Event::Park(pid)); } else { // An unpark landed in the prep-to-park window; the // word is back to Queued — re-queue instead of // parking. The lost-wakeup window, closed. crate::te!(crate::trace::Event::UnparkFlagConsumed(pid)); inner.enqueue(pid); } } } } } }