feat: I/O and mutex support (v0.3)

Add epoll-based non-blocking I/O and kernel-like mutexes:
- src/io.rs: Complete epoll backend with timeout & error handling
- src/mutex.rs: Fair mutex with waiter queues & parking integration
- Enhanced scheduler to support synchronous I/O blocking
- Comprehensive test suites for I/O (epoll) and mutex behavior
- Documentation: LOOM.md concurrency model & README

src/scheduler.rs

@@ -344,16 +344,69 @@ pub fn park_current() {
     unsafe { crate::context::switch_to_scheduler() };
 }
 
+/// RAII guard that disables allocator-driven preemption for its lifetime.
+///
+/// The "prep-to-park" hazard described in `preempt.rs`: a primitive that
+/// (a) registers an unparker (channel waiter slot, fd waiter map, mutex
+/// waiter queue, …) and then (b) calls `park_current()` must not yield
+/// between (a) and (b). If it does, an early unpark fires while the actor
+/// is still Runnable, the unpark no-ops, and then the actor parks with no
+/// one to wake it.
+///
+/// Library code wraps the prep + park in `let _g = NoPreempt::enter();`
+/// and the guard is held until just after `park_current` returns (or
+/// dropped earlier, immediately before `park_current`, since `park_current`
+/// itself returns control to the scheduler which disables preemption on
+/// its own path).
+pub struct NoPreempt(bool);
+
+impl NoPreempt {
+    pub fn enter() -> Self {
+        let prev = crate::preempt::PREEMPTION_ENABLED.with(|c| c.replace(false));
+        NoPreempt(prev)
+    }
+}
+
+impl Drop for NoPreempt {
+    fn drop(&mut self) {
+        crate::preempt::PREEMPTION_ENABLED.with(|c| c.set(self.0));
+    }
+}
+
 /// Park the current actor for at least `duration`. A zero duration behaves
 /// like `yield_now` (the deadline is immediately in the past, so the timer
 /// pops on the next scheduler iteration).
 pub fn sleep(duration: std::time::Duration) {
     let me = current_pid().expect("sleep() called outside an actor");
+    let _np = NoPreempt::enter();
     let deadline = crate::timer::deadline_from_now(duration);
-    with_sched(|s| s.timers.insert(deadline, me));
+    with_sched(|s| s.timers.insert_sleep(deadline, me));
     park_current();
 }
 
+/// Insert a `WaitTimeout` timer entry. Library code (`Mutex::lock_timeout`
+/// today, future bounded-wait primitives) calls this just before
+/// `park_current()` so that if the wait isn't satisfied by `deadline`,
+/// `target.on_timeout(pid, wait_seq)` will fire.
+///
+/// Cancellation: not needed. If the wait is satisfied early, the entry is
+/// still in the heap and will pop in due course; `on_timeout` is expected
+/// to be idempotent on stale-seq.
+pub fn insert_wait_timer(
+    deadline: std::time::Instant,
+    pid: Pid,
+    target: std::rc::Rc<dyn crate::timer::TimerTarget>,
+    wait_seq: u64,
+) {
+    with_sched(|s| {
+        s.timers.insert(
+            deadline,
+            pid,
+            crate::timer::Reason::WaitTimeout { target, wait_seq },
+        );
+    });
+}
+
 /// Run `f` on the IO worker thread, park the current actor while it runs,
 /// and return `f`'s value when it completes. Panics inside `f` propagate
 /// to the calling actor.
@@ -377,12 +430,14 @@ where
         Ok(Box::new(v) as Box<dyn std::any::Any + Send>)
     });
 
-    with_sched(|s| {
-        let io = s.io.as_mut().expect("io thread not started");
-        io.submit(me, work);
-    });
-
-    park_current();
+    {
+        let _np = NoPreempt::enter();
+        with_sched(|s| {
+            let io = s.io.as_mut().expect("io thread not started");
+            io.submit(me, work);
+        });
+        park_current();
+    }
 
     // On resume, our slot has a result waiting.
     let result = with_sched(|s| {
@@ -401,6 +456,83 @@ where
     }
 }
 
+// ---------------------------------------------------------------------------
+// Fd-readiness primitives.
+//
+// `wait_readable(fd)` / `wait_writable(fd)` register interest with the
+// epoll thread, park the calling actor, and return when the kernel
+// signals readiness. The subsequent syscall (`read`/`write`) is done on
+// the actor's own thread by the caller — no buffer crosses an actor
+// boundary.
+//
+// Fds passed in should be O_NONBLOCK; see io.rs module docs.
+// ---------------------------------------------------------------------------
+
+/// Park the calling actor until `fd` is readable.
+pub fn wait_readable(fd: std::os::fd::RawFd) -> std::io::Result<()> {
+    wait_fd(fd, /*readable=*/ true, /*writable=*/ false)
+}
+
+/// Park the calling actor until `fd` is writable.
+pub fn wait_writable(fd: std::os::fd::RawFd) -> std::io::Result<()> {
+    wait_fd(fd, /*readable=*/ false, /*writable=*/ true)
+}
+
+fn wait_fd(fd: std::os::fd::RawFd, readable: bool, writable: bool) -> std::io::Result<()> {
+    let me = current_pid().expect("wait_*() called outside an actor");
+
+    // Register with the epoll thread. If registration fails (bad fd,
+    // already-parked waiter, OOM in the kernel), return the error
+    // without parking — the actor never went to sleep.
+    let _np = NoPreempt::enter();
+    with_sched(|s| {
+        let io = s.io.as_mut().expect("io thread not started");
+        io.epoll_register(fd, me, readable, writable)
+    })?;
+
+    park_current();
+    // On resume, the scheduler has already removed `fd` from `waiters`
+    // and DEL'd it from epollfd. There is no per-call return value;
+    // success here just means "fd is ready, go do your syscall".
+    //
+    // Note: there is no error path on resume because v0.2 doesn't time
+    // out fd waits and doesn't otherwise spurious-wake. If those are
+    // added, this function grows a non-trivial return.
+    Ok(())
+}
+
+/// Wait until `fd` is readable, then run a single `read(2)`. Returns the
+/// number of bytes read, or an `io::Error` from the syscall.
+///
+/// `fd` should be opened `O_NONBLOCK`. With a blocking fd, the kernel's
+/// readiness signal does not guarantee a non-blocking read — a signal
+/// could interrupt, and the actor's syscall would then stall the
+/// scheduler thread.
+pub fn read(fd: std::os::fd::RawFd, buf: &mut [u8]) -> std::io::Result<usize> {
+    wait_readable(fd)?;
+    let n = unsafe {
+        libc::read(fd, buf.as_mut_ptr() as *mut _, buf.len())
+    };
+    if n < 0 {
+        Err(std::io::Error::last_os_error())
+    } else {
+        Ok(n as usize)
+    }
+}
+
+/// Wait until `fd` is writable, then run a single `write(2)`.
+pub fn write(fd: std::os::fd::RawFd, buf: &[u8]) -> std::io::Result<usize> {
+    wait_writable(fd)?;
+    let n = unsafe {
+        libc::write(fd, buf.as_ptr() as *const _, buf.len())
+    };
+    if n < 0 {
+        Err(std::io::Error::last_os_error())
+    } else {
+        Ok(n as usize)
+    }
+}
+
 /// Wake a parked actor. If the actor isn't parked (already runnable or done)
 /// this is a no-op — that's important; channel and join can both fire
 /// spurious unparks under some orderings and we want them to be cheap.
@@ -488,41 +620,95 @@ pub const ROOT_PID: Pid = Pid::new(u32::MAX, u32::MAX);
 
 fn schedule_loop() {
     loop {
-        // 1. Drain due timers into the run queue.
+        // 1. Drain due timers and dispatch by reason.
+        //
+        //   Sleep        — unpark the actor (idempotently: only if still
+        //                  parked).
+        //   WaitTimeout  — call the target's on_timeout. The target decides
+        //                  whether the wait was still in progress (timer
+        //                  won the race) or had been fulfilled (the thing
+        //                  the actor was waiting for arrived first → no-op).
+        //                  The target is responsible for calling unpark()
+        //                  if appropriate.
         let now = std::time::Instant::now();
         let due = with_sched(|s| s.timers.pop_due(now));
-        for pid in due {
-            // Same idempotency as `unpark`: only re-queue if still parked.
-            with_sched(|s| {
-                if let Some(slot) = s.slot_mut(pid) {
-                    if matches!(slot.state, State::Parked) {
-                        slot.state = State::Runnable;
-                        s.run_queue.push_back(pid);
-                    }
+        for entry in due {
+            match entry.reason {
+                crate::timer::Reason::Sleep => {
+                    with_sched(|s| {
+                        if let Some(slot) = s.slot_mut(entry.pid) {
+                            if matches!(slot.state, State::Parked) {
+                                slot.state = State::Runnable;
+                                s.run_queue.push_back(entry.pid);
+                            }
+                        }
+                    });
                 }
-            });
+                crate::timer::Reason::WaitTimeout { target, wait_seq } => {
+                    // Note: the target callback runs *outside* with_sched.
+                    // It may call back into the scheduler (e.g. unpark), so
+                    // we must not hold the SCHED borrow across it.
+                    target.on_timeout(entry.pid, wait_seq);
+                }
+            }
         }
 
-        // 2. Drain IO completions: route each result to its slot and
-        // unpark the actor. Drain even when we have other runnables —
-        // it's cheap (a try_lock of the completion queue) and keeps
-        // pending_io_result freshness bounded.
+        // 2. Drain IO completions: route each result by variant.
+        //
+        //   Blocking — a `block_on_io` closure finished. Stash the result
+        //              on the actor's slot and unpark.
+        //   FdReady  — an fd registered via `wait_readable`/`wait_writable`
+        //              is ready. Look up the parked pid in the io thread's
+        //              waiters map, deregister the fd, unpark.
+        //
+        // Drain even when we have other runnables — it's cheap and keeps
+        // `pending_io_result` / `waiters` freshness bounded.
         let completions = with_sched(|s| {
             s.io.as_mut().map(|io| io.drain_completions()).unwrap_or_default()
         });
-        for (pid, result) in completions {
-            with_sched(|s| {
-                if let Some(io) = s.io.as_mut() {
-                    io.outstanding = io.outstanding.saturating_sub(1);
+        for completion in completions {
+            match completion {
+                crate::io::Completion::Blocking { pid, result } => {
+                    with_sched(|s| {
+                        if let Some(io) = s.io.as_mut() {
+                            io.outstanding = io.outstanding.saturating_sub(1);
+                        }
+                        if let Some(slot) = s.slot_mut(pid) {
+                            slot.pending_io_result = Some(result);
+                            if matches!(slot.state, State::Parked) {
+                                slot.state = State::Runnable;
+                                s.run_queue.push_back(pid);
+                            }
+                        }
+                    });
                 }
-                if let Some(slot) = s.slot_mut(pid) {
-                    slot.pending_io_result = Some(result);
-                    if matches!(slot.state, State::Parked) {
-                        slot.state = State::Runnable;
-                        s.run_queue.push_back(pid);
-                    }
+                crate::io::Completion::FdReady { fd, events: _ } => {
+                    with_sched(|s| {
+                        let parked_pid = s.io.as_mut()
+                            .and_then(|io| {
+                                let pid = io.waiters.remove(&fd);
+                                // Deregister the fd from epollfd; the
+                                // EPOLLONESHOT already disarmed it but the
+                                // slot is still occupied until we DEL.
+                                io.epoll_deregister(fd);
+                                pid
+                            });
+                        if let Some(pid) = parked_pid {
+                            if let Some(slot) = s.slot_mut(pid) {
+                                if matches!(slot.state, State::Parked) {
+                                    slot.state = State::Runnable;
+                                    s.run_queue.push_back(pid);
+                                }
+                            }
+                            // else: actor died between registering and the
+                            // fd firing. Nothing to do; the registration
+                            // has been cleaned up.
+                        }
+                        // else: fd not in waiters — probably a duplicate
+                        // FdReady from a previous registration, ignore.
+                    });
                 }
-            });
+            }
         }
 
         // 3. Pop a runnable actor. If none, decide whether to block on
@@ -536,11 +722,19 @@ fn schedule_loop() {
                 // trying to take the completions mutex, which is fine,
                 // but the scheduler thread itself mustn't hold SCHED
                 // borrowed across a blocking syscall.
+                //
+                // "Outstanding" here means *anything* the IO thread is
+                // expected to deliver a wakeup for: in-flight blocking
+                // calls AND parked fd waiters. If either is non-zero we
+                // must wait for the IO thread, not exit.
                 let (next_deadline, io_outstanding, wake_fd) = with_sched(|s| {
                     let next = s.timers.peek_deadline();
                     let (out, fd) = match s.io.as_ref() {
-                        Some(io) => (io.outstanding, Some(io.wake_fd())),
-                        None     => (0, None),
+                        Some(io) => (
+                            io.outstanding + io.waiters.len() as u32,
+                            Some(io.wake_fd()),
+                        ),
+                        None => (0, None),
                     };
                     (next, out, fd)
                 });