//! The run queue, selected at COMPILE TIME by mutually-exclusive cargo //! features (no runtime dispatch — the scheduler's pop loop is the hottest //! code in the runtime): //! //! - `rq-mutex` (default) — `Mutex`. The control/baseline: //! strictly FIFO, trivially correct, one global lock. //! - `rq-mpmc` — a single hand-rolled Vyukov bounded MPMC ring (per-cell //! sequence numbers). Strict FIFO, lock-free, one hot //! enqueue/dequeue cache-line pair. //! - `rq-striped` — M Vyukov rings with fetch-add ticket distribution. //! *Relaxed* FIFO: ordering across stripes is bounded-skewed //! (≈ one ring's worth of reordering per stripe), in exchange //! for spreading the hot line M ways. Predicted winner at //! high core counts; phase 4's shootout decides. //! //! Select non-default variants with `--no-default-features --features rq-…` //! (cargo features are additive, so the default must be switched off). //! //! All variants are compiled unconditionally (so every build runs every //! variant's unit tests); the feature only picks which one the runtime uses //! via the [`RunQueue`] alias. //! //! # Contract (shared by all variants) //! //! - **Occupancy is bounded by `max_actors`.** A pid is in the queue at most //! once (pushes pair 1:1 with transitions into `Queued`; only the //! scheduler transitions `Queued → Running` — see the state-machine docs //! in `runtime.rs`), and at most `max_actors` actors exist. With the RFC //! 005 wake slot enabled the invariant reads "in (slot ⊕ shared queue) at //! most once" — a slot push *replaces* the queue push at the same //! protocol point, and a displacement moves the occupant, never copies //! it — so the bound holds verbatim. The bounded rings are sized ≥ //! `max_actors`, so **`push` is infallible**; a full //! ring is an invariant violation and panics loudly rather than spinning. //! - **Preemption must be disabled around every push/pop** (debug-asserted). //! For the mutex variant this is the usual no-switch/no-unwind-under-lock //! rule. For the rings it is *load-bearing in a sharper way*: a producer //! suspended between claiming a cell and publishing its sequence number //! stalls every consumer behind that cell — on a busy runtime that is a //! livelock, since the suspended actor's own resume entry sits behind the //! hole. Callers get this for free: every queue op happens inside //! `with_runtime`/`try_with_runtime` (NoPreempt for their span since //! phase 2) or on a scheduler thread between resumes (preemption off). //! - **`pop() == None` is a snapshot, not a fence.** A push that is mid- //! publish (or in a stripe the probe already passed) may be missed; the //! caller's idle path sleeps ≤ 100µs and retries, so the cost is a bounded //! latency blip, never a lost entry. Termination does not lean on this: //! the all-clear is `live_actors == 0` (+ io quiescent), and `live == 0` //! already implies the queue holds nothing actionable — see the argument //! in `schedule_loop`. //! - `len()` is approximate (stats only). use crate::pid::Pid; use crate::sync_shim::{AtomicUsize, Ordering, UnsafeCell}; use std::mem::MaybeUninit; // --------------------------------------------------------------------------- // Feature selection // --------------------------------------------------------------------------- #[cfg(not(any(feature = "rq-mutex", feature = "rq-mpmc", feature = "rq-striped")))] compile_error!( "smarm: no run queue selected. Enable exactly one of the features \ `rq-mutex` (default), `rq-mpmc`, `rq-striped`." ); #[cfg(all(feature = "rq-mutex", feature = "rq-mpmc"))] compile_error!( "smarm: features `rq-mutex` and `rq-mpmc` are mutually exclusive \ (use --no-default-features to drop the default `rq-mutex`)." ); #[cfg(all(feature = "rq-mutex", feature = "rq-striped"))] compile_error!( "smarm: features `rq-mutex` and `rq-striped` are mutually exclusive \ (use --no-default-features to drop the default `rq-mutex`)." ); #[cfg(all(feature = "rq-mpmc", feature = "rq-striped"))] compile_error!("smarm: features `rq-mpmc` and `rq-striped` are mutually exclusive."); #[cfg(feature = "rq-mutex")] pub(crate) type RunQueue = MutexQueue; #[cfg(feature = "rq-mpmc")] pub(crate) type RunQueue = MpmcRing; #[cfg(feature = "rq-striped")] pub(crate) type RunQueue = StripedRing; #[inline] fn assert_no_preempt() { debug_assert!( !crate::preempt::PREEMPTION_ENABLED.with(|c| c.get()), "run-queue op with preemption enabled — a switch mid-op stalls or \ corrupts the queue; route through with_runtime or scheduler context" ); } // --------------------------------------------------------------------------- // rq-mutex — the baseline // --------------------------------------------------------------------------- #[allow(dead_code)] pub struct MutexQueue { q: std::sync::Mutex>, } #[allow(dead_code)] impl MutexQueue { pub fn new(_threads: usize, max_actors: usize) -> Self { Self { // Pre-size: the queue can never outgrow the slab, and one // allocation at init beats reallocating under the lock later. q: std::sync::Mutex::new(std::collections::VecDeque::with_capacity(max_actors)), } } pub fn push(&self, pid: Pid) { assert_no_preempt(); match self.q.lock() { Ok(mut g) => g.push_back(pid), Err(e) => panic!("smarm: run-queue q lock poisoned (core corrupt): {e}"), } } pub fn pop(&self) -> Option { assert_no_preempt(); match self.q.lock() { Ok(mut g) => g.pop_front(), Err(e) => panic!("smarm: run-queue q lock poisoned (core corrupt): {e}"), } } pub fn len(&self) -> u64 { match self.q.lock() { Ok(g) => g.len() as u64, Err(e) => panic!("smarm: run-queue q lock poisoned (core corrupt): {e}"), } } pub fn is_empty(&self) -> bool { match self.q.lock() { Ok(g) => g.is_empty(), Err(e) => panic!("smarm: run-queue q lock poisoned (core corrupt): {e}"), } } } // --------------------------------------------------------------------------- // rq-mpmc — Vyukov bounded MPMC ring // --------------------------------------------------------------------------- // // Dmitry Vyukov's bounded MPMC queue: each cell carries a sequence number. // A producer may write cell `i` when `seq == pos` (its turn); it publishes // with `seq = pos + 1`. A consumer may read when `seq == pos + 1`; it // releases the cell to the next lap with `seq = pos + capacity`. Producers // and consumers each contend on one counter; cell handoff is a per-cell // Acquire/Release pair, so unrelated push/pop pairs don't serialize. /// Pad to a cache-line pair so the producer and consumer counters (and the /// cells) don't false-share. #[repr(align(128))] struct CachePadded(T); struct Cell { seq: AtomicUsize, pid: UnsafeCell>, } #[allow(dead_code)] pub struct MpmcRing { buf: Box<[Cell]>, mask: usize, enqueue_pos: CachePadded, dequeue_pos: CachePadded, } // SAFETY: cells are handed off between threads via the per-cell seq // (Release on publish, Acquire on claim); Pid is Copy + Send. unsafe impl Send for MpmcRing {} unsafe impl Sync for MpmcRing {} #[allow(dead_code)] impl MpmcRing { pub fn new(_threads: usize, max_actors: usize) -> Self { Self::with_capacity(max_actors) } /// Pub for the raw-structure microbench (sized to the op count so the /// occupancy contract is trivially met there). Runtime code uses `new`. pub fn with_capacity(min_cap: usize) -> Self { // Occupancy ≤ max_actors (queue contract), so capacity = the next // power of two ≥ max_actors can never overflow. (≥ 2 so mask works.) let cap = min_cap.next_power_of_two().max(2); let buf: Box<[Cell]> = (0..cap) .map(|i| Cell { seq: AtomicUsize::new(i), pid: UnsafeCell::new(MaybeUninit::uninit()), }) .collect(); Self { buf, mask: cap - 1, enqueue_pos: CachePadded(AtomicUsize::new(0)), dequeue_pos: CachePadded(AtomicUsize::new(0)), } } pub fn push(&self, pid: Pid) { assert_no_preempt(); assert!( self.try_push(pid), "smarm: run queue overflow — occupancy exceeded the slab bound, \ which the at-most-once-enqueued invariant forbids. This is a \ runtime bug (double enqueue), not a capacity tuning problem." ); } /// One full claim attempt; `false` only if the ring is full. fn try_push(&self, pid: Pid) -> bool { let mut pos = self.enqueue_pos.0.load(Ordering::Relaxed); loop { let cell = &self.buf[pos & self.mask]; let seq = cell.seq.load(Ordering::Acquire); let diff = seq as isize - pos as isize; if diff == 0 { // Our turn: claim the position. match self.enqueue_pos.0.compare_exchange_weak( pos, pos + 1, Ordering::Relaxed, Ordering::Relaxed, ) { Ok(_) => { // SAFETY: the claim gives us exclusive write access // to this cell until we publish below. cell.pid.with_mut(|p| unsafe { (*p).write(pid) }); cell.seq.store(pos + 1, Ordering::Release); return true; } Err(actual) => pos = actual, } } else if diff < 0 { return false; // full — a whole lap behind } else { pos = self.enqueue_pos.0.load(Ordering::Relaxed); } } } pub fn pop(&self) -> Option { assert_no_preempt(); let mut pos = self.dequeue_pos.0.load(Ordering::Relaxed); loop { let cell = &self.buf[pos & self.mask]; let seq = cell.seq.load(Ordering::Acquire); let diff = seq as isize - (pos + 1) as isize; if diff == 0 { match self.dequeue_pos.0.compare_exchange_weak( pos, pos + 1, Ordering::Relaxed, Ordering::Relaxed, ) { Ok(_) => { // SAFETY: the claim gives us exclusive read access; // the producer's Release publish made `pid` visible // to our Acquire load of `seq`. let pid = cell.pid.with(|p| unsafe { (*p).assume_init_read() }); // Release the cell for the next lap. cell.seq.store(pos + self.mask + 1, Ordering::Release); return Some(pid); } Err(actual) => pos = actual, } } else if diff < 0 { // Empty (or the producer at this cell hasn't published yet — // a snapshot miss the caller's idle-retry loop absorbs). return None; } else { pos = self.dequeue_pos.0.load(Ordering::Relaxed); } } } pub fn len(&self) -> u64 { let e = self.enqueue_pos.0.load(Ordering::Relaxed); let d = self.dequeue_pos.0.load(Ordering::Relaxed); e.saturating_sub(d) as u64 } pub fn is_empty(&self) -> bool { self.len() == 0 } } // --------------------------------------------------------------------------- // rq-striped — M Vyukov rings, ticket-distributed // --------------------------------------------------------------------------- // // Producers fetch-add a ticket and start probing at stripe `ticket % M`; // consumers do the same with their own ticket. Under symmetric load the // tickets spread producers and consumers uniformly, so each stripe sees // ~1/M of the traffic and the single hot cache-line pair becomes M cooler // ones. FIFO is relaxed: two pushes that land in different stripes can be // popped in either order, with skew bounded by stripe occupancy imbalance. // // Push probes forward from its home stripe until a `try_push` succeeds. // Σ stripe capacity ≥ 2 × max_actors while occupancy ≤ max_actors, so at // every instant at least half the total capacity is free and the probe // terminates (in practice on the first stripe). #[allow(dead_code)] pub struct StripedRing { stripes: Box<[MpmcRing]>, /// Stripe count minus one (count is a power of two). stripe_mask: usize, push_ticket: CachePadded, pop_ticket: CachePadded, } #[allow(dead_code)] impl StripedRing { pub fn new(threads: usize, max_actors: usize) -> Self { // One stripe per scheduler thread, rounded up to a power of two — // more stripes than threads buys nothing (at most `threads` ops are // in flight) and costs pop-probe latency when mostly empty. let n = threads.max(1).next_power_of_two(); // Per-stripe capacity: 2 × max_actors / n in total, and never below // a floor that keeps degenerate configs (tiny slab, many threads) // trivially correct. let per = ((2 * max_actors) / n).next_power_of_two().max(8); let stripes: Box<[MpmcRing]> = (0..n).map(|_| MpmcRing::with_capacity(per)).collect(); Self { stripes, stripe_mask: n - 1, push_ticket: CachePadded(AtomicUsize::new(0)), pop_ticket: CachePadded(AtomicUsize::new(0)), } } pub fn push(&self, pid: Pid) { assert_no_preempt(); let home = self.push_ticket.0.fetch_add(1, Ordering::Relaxed); // Probe from the home stripe; capacity headroom (Σ ≥ 2×occupancy) // guarantees a free stripe exists, so the outer loop terminates. // The retry-from-home lap handles the racy case where every stripe // momentarily refused us. loop { for i in 0..=self.stripe_mask { let s = &self.stripes[(home + i) & self.stripe_mask]; if s.try_push(pid) { return; } } std::hint::spin_loop(); } } pub fn pop(&self) -> Option { assert_no_preempt(); let home = self.pop_ticket.0.fetch_add(1, Ordering::Relaxed); for i in 0..=self.stripe_mask { if let Some(pid) = self.stripes[(home + i) & self.stripe_mask].pop() { return Some(pid); } } None // snapshot miss possible across stripes; idle-retry absorbs it } pub fn len(&self) -> u64 { self.stripes.iter().map(|s| s.len()).sum() } pub fn is_empty(&self) -> bool { self.stripes.iter().all(|s| s.is_empty()) } } // --------------------------------------------------------------------------- // Tests — all variants, in every build (the feature only picks the alias) // --------------------------------------------------------------------------- #[cfg(all(test, not(loom)))] mod tests { use super::*; use std::collections::HashSet; use std::sync::Arc; fn pid(i: u32) -> Pid { Pid::new(i, 0) } fn fifo_smoke(q: &Q, push: impl Fn(&Q, Pid), pop: impl Fn(&Q) -> Option) { for i in 0..100 { push(q, pid(i)); } for i in 0..100 { assert_eq!(pop(q), Some(pid(i))); } assert_eq!(pop(q), None); } #[test] fn mutex_fifo() { let q = MutexQueue::new(1, 1024); fifo_smoke(&q, |q, p| q.push(p), |q| q.pop()); } #[test] fn mpmc_fifo_single_thread() { let q = MpmcRing::new(1, 1024); fifo_smoke(&q, |q, p| q.push(p), |q| q.pop()); } #[test] fn mpmc_wraps_many_laps() { let q = MpmcRing::with_capacity(8); for lap in 0..1000u32 { for i in 0..8 { q.push(pid(lap * 8 + i)); } for i in 0..8 { assert_eq!(q.pop(), Some(pid(lap * 8 + i))); } } assert_eq!(q.pop(), None); } /// N producers, M consumers, every element exactly once. Run on plain OS /// threads (PREEMPTION_ENABLED defaults false, satisfying the contract). fn exactly_once( q: Q, push: fn(&Q, Pid), pop: fn(&Q) -> Option, producers: u32, consumers: u32, per_producer: u32, ) { let q = Arc::new(q); let total = (producers * per_producer) as usize; let popped = Arc::new(std::sync::Mutex::new(Vec::with_capacity(total))); let remaining = Arc::new(AtomicUsize::new(total)); let mut hs = Vec::new(); for p in 0..producers { let q = q.clone(); hs.push(std::thread::spawn(move || { for i in 0..per_producer { push(&q, pid(p * per_producer + i)); } })); } for _ in 0..consumers { let q = q.clone(); let popped = popped.clone(); let remaining = remaining.clone(); hs.push(std::thread::spawn(move || { let mut local = Vec::new(); while remaining.load(Ordering::Relaxed) > 0 { if let Some(pid) = pop(&q) { remaining.fetch_sub(1, Ordering::Relaxed); local.push(pid); } else { std::hint::spin_loop(); } } popped.lock().unwrap().extend(local); })); } for h in hs { h.join().unwrap(); } let popped = popped.lock().unwrap(); assert_eq!(popped.len(), total, "count mismatch"); let set: HashSet = popped.iter().map(|p| ((p.index() as u64) << 32) | p.generation() as u64).collect(); assert_eq!(set.len(), total, "duplicate or lost element"); assert_eq!(pop(&q), None); } #[test] fn mpmc_exactly_once_contended() { exactly_once(MpmcRing::new(8, 4096), |q, p| q.push(p), |q| q.pop(), 4, 4, 1000); } #[test] fn striped_exactly_once_contended() { exactly_once(StripedRing::new(8, 4096), |q, p| q.push(p), |q| q.pop(), 4, 4, 1000); } #[test] fn striped_drains_after_skewed_load() { // Hammer pushes from one thread (all tickets walk the stripes in // order) and verify a single consumer sees every element. let q = StripedRing::new(4, 64); let mut seen = HashSet::new(); for i in 0..64 { q.push(pid(i)); } while let Some(p) = q.pop() { assert!(seen.insert(p.index())); } assert_eq!(seen.len(), 64); } } // --------------------------------------------------------------------------- // loom model tests — RUSTFLAGS="--cfg loom" cargo test --lib --release // --------------------------------------------------------------------------- #[cfg(all(test, loom))] mod loom_tests { use super::*; use loom::sync::Arc; use loom::thread; fn pid(i: u32) -> Pid { Pid::new(i, 0) } /// Two producers, main-thread consumer: both elements arrive exactly /// once, across every interleaving — including through a lap wraparound /// (capacity 2 forces cell reuse). #[test] fn mpmc_two_producers_exactly_once() { loom::model(|| { let q = Arc::new(MpmcRing::with_capacity(2)); let mut hs = Vec::new(); for i in 0..2u32 { let q = q.clone(); hs.push(thread::spawn(move || q.push(pid(i)))); } let mut got = Vec::new(); while got.len() < 2 { match q.pop() { Some(p) => got.push(p.index()), None => thread::yield_now(), } } for h in hs { h.join().unwrap(); } got.sort_unstable(); assert_eq!(got, vec![0, 1]); assert!(q.pop().is_none()); }); } /// Producer races a consumer on a single element: the consumer either /// gets it or sees a clean None — never a torn/duplicated element. #[test] fn mpmc_push_pop_race() { loom::model(|| { let q = Arc::new(MpmcRing::with_capacity(2)); let q2 = q.clone(); let prod = thread::spawn(move || q2.push(pid(7))); let seen = q.pop(); prod.join().unwrap(); match seen { Some(p) => { assert_eq!(p.index(), 7); assert!(q.pop().is_none()); } None => assert_eq!(q.pop().map(|p| p.index()), Some(7)), } }); } /// Striped: two producers landing in (potentially) different stripes, /// main-thread consumer drains both exactly once. #[test] fn striped_two_producers_exactly_once() { loom::model(|| { let q = Arc::new(StripedRing::new(2, 4)); let mut hs = Vec::new(); for i in 0..2u32 { let q = q.clone(); hs.push(thread::spawn(move || q.push(pid(i)))); } let mut got = Vec::new(); while got.len() < 2 { match q.pop() { Some(p) => got.push(p.index()), None => thread::yield_now(), } } for h in hs { h.join().unwrap(); } got.sort_unstable(); assert_eq!(got, vec![0, 1]); assert!(q.pop().is_none()); }); } }