fix(tls): actor-side thread-local accessors are #[inline(never)] + fence — LLVM caches TLS addresses across context switches (finding 19)
Without LTO (any downstream `cargo build --release`), LLVM keeps `%fs:0` in a
callee-saved register across `switch_to_scheduler`; after the actor migrates,
`ACTOR_DONE`/`PREEMPTION_ENABLED`/`CURRENT_PID` etc. hit the OLD thread's TLS.
8 multi-thread tests aborted with "scheduler resumed a done actor" (error 5).
Thin LTO only worked because it picked the local-exec model.
- context.rs: module doc "Thread-locals and migration" (the rule), tls_fence()
- every TLS accessor reachable from an actor stack is out of line:
actor::{current_pid, publish_outcome (new)}, preempt::{maybe_preempt,
check_cancelled, current_slot_ptr, note_*, preemption_swap/enabled (new;
replaces direct PREEMPTION_ENABLED.with in NoPreempt/RawMutex/with_runtime/
trace/debug asserts)}, context::{get,set}_scheduler_sp, runtime::{sched_slot,
slot_push, set_yield_intent}, causal, trace
- raw_mutex order checks: out of line only under debug_assertions
- Cargo.toml: [profile.reltest] = release without LTO; 41 test bins link in
~1.5 s each instead of ~12 s (suite 11 min -> 1.5 min) AND it is the
regression oracle for this bug: `cargo test --profile reltest`
Both profiles: 41/41 + doctests green.
This commit is contained in:
+17
-7
@@ -418,7 +418,7 @@ impl SchedulerStats {
|
||||
/// Bump a per-thread diagnostic counter on the calling scheduler thread.
|
||||
macro_rules! diag {
|
||||
($inner:expr, $field:ident) => {
|
||||
SCHED_SLOT.with(|s| $inner.stats[s.get()].$field.fetch_add(1, Ordering::Relaxed))
|
||||
$inner.stats[sched_slot()].$field.fetch_add(1, Ordering::Relaxed)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1239,9 +1239,11 @@ impl RuntimeInner {
|
||||
/// Displacement: the NEW wake takes the slot (newest is hottest; the old
|
||||
/// occupant was about to lose its locality window anyway) and the old
|
||||
/// occupant is pushed to the shared queue.
|
||||
#[inline(never)]
|
||||
fn slot_push(&self, pid: Pid) {
|
||||
crate::context::tls_fence();
|
||||
debug_assert!(
|
||||
!crate::preempt::PREEMPTION_ENABLED.with(|c| c.get()),
|
||||
!crate::preempt::preemption_enabled(),
|
||||
"slot_push with preemption enabled — a switch mid-op could \
|
||||
migrate the actor and split the slot access across threads"
|
||||
);
|
||||
@@ -1255,11 +1257,9 @@ impl RuntimeInner {
|
||||
let displaced = WAKE_SLOT.with(|s| s.replace(Some(pid)));
|
||||
crate::te!(crate::trace::Event::SlotPush(pid));
|
||||
if let Some(old) = displaced {
|
||||
SCHED_SLOT.with(|s| {
|
||||
self.stats[s.get()]
|
||||
.slot_displacements
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
});
|
||||
self.stats[sched_slot()]
|
||||
.slot_displacements
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
self.enqueue(old);
|
||||
}
|
||||
}
|
||||
@@ -1640,7 +1640,17 @@ pub(crate) enum YieldIntent {
|
||||
Park,
|
||||
}
|
||||
|
||||
/// This scheduler thread's stats index. Read from actor context by every
|
||||
/// `stat!()` on the enqueue/wake paths, hence out of line (`context` docs).
|
||||
#[inline(never)]
|
||||
fn sched_slot() -> usize {
|
||||
crate::context::tls_fence();
|
||||
SCHED_SLOT.with(|s| s.get())
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
pub(crate) fn set_yield_intent(i: YieldIntent) {
|
||||
crate::context::tls_fence();
|
||||
YIELD_INTENT.with(|c| c.set(i));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user