diff --git a/src/causal.rs b/src/causal.rs index 34a7c52..89f3155 100644 --- a/src/causal.rs +++ b/src/causal.rs @@ -279,7 +279,9 @@ mod inner { if slot.is_null() { return; } - let now = preempt::rdtsc(); + // Serialised: `now` closes an interval attributed to a code site; a + // speculative early read would drop that site's tail (RFC 007). + let now = preempt::rdtsc_serialising(); let last = LAST_SAMPLE_TSC.with(|c| c.replace(now)); let target_site = (exp >> 32) as u32; let pct = exp & 0xffff_ffff; @@ -374,7 +376,8 @@ mod inner { let target = (exp >> 32) as u32; let pct = exp & 0xffff_ffff; if old == target && new != target { - let now = preempt::rdtsc(); + // Serialised: guard exit bounds the site's interval exactly. + let now = preempt::rdtsc_serialising(); let last = LAST_SAMPLE_TSC.with(|c| c.replace(now)); if pct > 0 { if last != 0 { @@ -386,7 +389,9 @@ mod inner { } } } else if new == target && old != target { - LAST_SAMPLE_TSC.with(|c| c.set(preempt::rdtsc())); + // Serialised: an early arm would let pre-site work leak into the + // first in-site interval — the over-attribution this guards. + LAST_SAMPLE_TSC.with(|c| c.set(preempt::rdtsc_serialising())); } } diff --git a/src/preempt.rs b/src/preempt.rs index f7d2796..d11d0db 100644 --- a/src/preempt.rs +++ b/src/preempt.rs @@ -198,8 +198,27 @@ pub(crate) fn elapsed_slice_cycles() -> u64 { rdtsc().saturating_sub(TIMESLICE_START.with(|c| c.get())) } +/// Read the TSC, unserialised. The core may execute this before earlier +/// instructions retire (or after later ones start), so a single stamp can +/// land tens of cycles — worst case a stalled load's worth — early or late. +/// That is negligible against every consumer in this module: the timeslice +/// arm/expiry compare against a ~10^5-cycle slice, and an early stamp only +/// makes a slice look *more* used (expires marginally sooner, never later). +/// The `lfence` this used to carry was a pipeline drain paid on every +/// resume; the one place that needs it is causal-site attribution, which +/// opts in via [`rdtsc_serialising`]. #[inline(always)] pub fn rdtsc() -> u64 { + // SAFETY: x86-64 only (this crate is x86-64 Linux only). + unsafe { core::arch::x86_64::_rdtsc() } +} + +/// Read the TSC after all prior instructions have completed locally. +/// Use where a stamp bounds an interval attributed to *code* — a speculative +/// early read would credit the tail of that code to whatever comes next. +/// Costs a pipeline drain; keep it off the per-resume path. +#[inline(always)] +pub fn rdtsc_serialising() -> u64 { unsafe { // SAFETY: x86-64 only. `lfence` serialises the instruction stream so // we don't measure time before prior instructions retire.