From dcef6f1f2fe916bf44abd39c786acff36410eac5 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 3 Aug 2026 10:34:34 +0200 Subject: [PATCH 01/19] WIP: I/O API + simulator --- Cargo.lock | 4 + crates/runtime-core/Cargo.toml | 4 +- crates/runtime-core/src/io/mod.rs | 148 ++++++++++++ crates/runtime-core/src/lib.rs | 2 + crates/runtime-core/src/sim/io/fs.rs | 154 ++++++++++++ crates/runtime-core/src/sim/io/mod.rs | 261 +++++++++++++++++++++ crates/runtime-core/src/sim/io/op.rs | 322 ++++++++++++++++++++++++++ crates/runtime-core/src/sim/mod.rs | 1 + crates/runtime/Cargo.toml | 4 + crates/runtime/src/io.rs | 2 + crates/runtime/src/io/tokio.rs | 163 +++++++++++++ crates/runtime/src/lib.rs | 2 + 12 files changed, 1066 insertions(+), 1 deletion(-) create mode 100644 crates/runtime-core/src/io/mod.rs create mode 100644 crates/runtime-core/src/sim/io/fs.rs create mode 100644 crates/runtime-core/src/sim/io/mod.rs create mode 100644 crates/runtime-core/src/sim/io/op.rs create mode 100644 crates/runtime/src/io.rs create mode 100644 crates/runtime/src/io/tokio.rs diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..0289ef17a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8333,7 +8333,9 @@ dependencies = [ "futures", "libc", "spacetimedb-runtime-core", + "static_assertions", "tokio", + "windows-sys 0.61.2", ] [[package]] @@ -8341,7 +8343,9 @@ name = "spacetimedb-runtime-core" version = "2.8.3" dependencies = [ "async-task", + "futures-channel", "spin", + "zerocopy", ] [[package]] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index a3369a69f89..6ac037162dd 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -11,8 +11,10 @@ workspace = true [features] default = [] -sim = ["dep:async-task", "dep:spin"] +sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } +futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } +zerocopy = "0.8" diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs new file mode 100644 index 00000000000..b4ff3744916 --- /dev/null +++ b/crates/runtime-core/src/io/mod.rs @@ -0,0 +1,148 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +/// Size in bytes of a disk sector. +pub const SECTOR_SIZE: usize = 4096; + +/// Types that can be safely converted to and from sector-aligned byte slices. +pub trait AlignedBytes: Sized { + /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the + /// right aligment. + /// + /// NOTE: Associated constants are evaluated lazily -- add a free + /// + /// `const _: () = ::ASSERT_VALID_LAYOUT;` + /// + /// for each `T` that is supposed to be used as an `AlignedBytes`. + const ASSERT_VALID_LAYOUT: () = { + assert!(align_of::() == SECTOR_SIZE); + assert!(size_of::().is_multiple_of(SECTOR_SIZE)); + }; + + /// Reinterpret `self` as a byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes(&self) -> &[u8]; + + /// Reinterpret `self` as a mutable byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes_mut(&mut self) -> &mut [u8]; + + /// Reinterpret a byte slice as `Self`. + /// + /// The slice must be of length `size_of::()`. + /// + /// NOTE: Any slice of the right size, but consisting of only `0` (zero) + /// bytes can be converted to `Self`. It is the caller's responsibility to + /// validate the returned type as per the application's invariants. + /// + /// # Panics + /// + /// Panics if `b.len() != size_of::()`. + fn from_bytes(b: &[u8]) -> Self; +} + +impl AlignedBytes for T { + fn as_bytes(&self) -> &[u8] { + ::as_bytes(self) + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + ::as_mut_bytes(self) + } + + fn from_bytes(b: &[u8]) -> Self { + Self::read_from_bytes(b).unwrap() + } +} + +/// An error `E`, along with auxiliary data `T`. +/// +/// `T` is usually a buffer of type [AlignedBytes], whose ownership is +/// transferred back to the caller when an error occurs. +/// +/// As this type signifies an error condition, the contents of `T` are +/// unspecified. +#[derive(Debug)] +pub struct ErrorWith { + pub error: E, + pub with: T, +} + +/// The canonical, low-level I/O API. +/// +/// Currently only supports file I/O, but eventually all I/O performed by +/// SpacetimeDB should go through this trait. +/// +/// Intended to support implementations based on `io-uring`, which means that +/// buffer ownership is transferred to the I/O engine while reading or writing. +/// +/// Implementations should be `!Send`, i.e. all I/O happens on a single thread. +/// +/// File operations should never be mutually exclusive, and therefore expose a +/// `pwrite`/`pread`-style API. It is assumed that direct I/O (`O_DIRECT`) is +/// used, i.e. the kernel page cache is bypassed. The [AlignedBytes] type +/// ensures that the alignment requirements for direct I/O are met. +pub trait SpacetimeIO { + /// An open file handle. + /// + /// Like [std::fs::File], the file shall be closed when the last reference + /// to the handle is dropped. + /// + /// Unlike [std::fs::File], the file handle must be clone-able. + type Fd: Clone; + /// The error returned by methods of this trait. + /// + /// This should always be instantiated to [std::io::Error]. However, pending + /// [alloc_io], this type is not in `core`, which would prevent this crate + /// from being `no_std`. + /// + /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 + type Error; + + /// Open the file at `path`. + fn open_file(&self, path: &str) -> impl Future>; + + /// Create the file at `path` and allocate `len` bytes. + /// + /// Returns an error if the file already exists. + fn create_file(&self, path: &str, len: u64) -> impl Future>; + + /// Write `buf` to `fd` at `offset`. + /// + /// `offset` must be a multiple of [SECTOR_SIZE]. + /// + /// Behaves like `FileExt::write_all_at`, i.e. tries to write all bytes in + /// `buf`, potentially retrying on errors of kind interrupted, and returns + /// an error if that fails. + fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> impl Future>>; + + /// Read `size_of::()` bytes from `fd` at `offset` and interpret them at + /// type `B`. + /// + /// `offset` must be a multiple of [SECTOR_SIZE]. + /// + /// Behaves like `FileExt::read_exact_at`, i.e. attempts to read + /// `size_of::()` bytes, potentially retrying on errors of kind + /// interrupted, and returns an error if less than the required bytes could + /// be read. + fn read_exact_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> impl Future>>; + + /// Call `fsync(2)` on `fd`. + fn fsync(&self, fd: Self::Fd) -> impl Future>; + /// Call `fdatasync(2)` on `fd`. + fn fdatasync(&self, fd: Self::Fd) -> impl Future>; + + /// Allocate `additional` bytes for the file `fd`. + fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; +} diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index f7590ada98b..e35d042ea9a 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -7,3 +7,5 @@ extern crate std; #[cfg(feature = "sim")] pub mod sim; + +pub mod io; diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs new file mode 100644 index 00000000000..9c540f5e1b3 --- /dev/null +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -0,0 +1,154 @@ +use alloc::{collections::BTreeMap, rc::Rc}; +use core::{ + cell::{Cell, RefCell}, + cmp, +}; + +pub const PAGE_SIZE: usize = 4096; +const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Error { + UnalignedOffset, + UnalignedBuffer, + OffsetOverflow, +} + +pub type Result = core::result::Result; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct PageIndex(u64); + +impl PageIndex { + fn from_offset(offset: u64) -> Self { + assert!(offset.is_multiple_of(PAGE_SIZE_U64)); + Self(offset / PAGE_SIZE_U64) + } +} + +struct Page { + bytes: RefCell<[u8; PAGE_SIZE]>, +} + +impl Page { + fn zeroed() -> Self { + Self { + bytes: RefCell::new([0; PAGE_SIZE]), + } + } +} + +/// A memory-backed file. +/// +/// A [File] is backed by a sparse array of [Page]s. Missing pages are read as +/// zeroes. +/// +/// Read and write operations must be page-aligned. Only full pages can be read +/// or written. Writing a page is atomic. +#[derive(Clone)] +pub struct File { + pages: RefCell>>, + len: Cell, +} + +impl File { + pub(super) const fn new() -> Self { + Self { + pages: RefCell::new(BTreeMap::new()), + len: Cell::new(0), + } + } + + pub(super) const fn len(&self) -> u64 { + self.len.get() + } + + #[allow(unused)] + pub(super) const fn is_empty(&self) -> bool { + self.len.get() == 0 + } + + /// Change the file length. + /// + /// The new length must be page-aligned. + /// + /// Extending allocates pages eagerly as needed. Shrinking drops all pages + /// at or beyond the new EOF. + pub fn set_len(&self, new_len: u64) -> Result<()> { + use cmp::Ordering::*; + + if !new_len.is_multiple_of(PAGE_SIZE_U64) { + return Err(Error::UnalignedOffset); + } + let old_len = self.len.get(); + + match new_len.cmp(&old_len) { + Equal => {} + Greater => { + let first_new_page = old_len / PAGE_SIZE_U64; + let end_page = new_len / PAGE_SIZE_U64; + + for index in first_new_page..end_page { + self.get_or_allocate_page(PageIndex(index)); + } + + self.len.set(new_len); + } + Less => { + self.len.set(new_len); + + let first_removed = PageIndex::from_offset(new_len); + let removed = self.pages.borrow_mut().split_off(&first_removed); + drop(removed); + } + } + + Ok(()) + } + + /// Read one complete page. + pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { + if dst.len() != PAGE_SIZE { + return Err(Error::UnalignedBuffer); + } + + match self.get_page(PageIndex(index)) { + Some(page) => { + dst.copy_from_slice(&*page.bytes.borrow()); + } + None => { + dst.fill(0); + } + } + + Ok(()) + } + + /// Write one complete page. + pub fn write_page(&self, src: &[u8], index: u64) -> Result<()> { + if src.len() != PAGE_SIZE { + return Err(Error::UnalignedBuffer); + } + + let page = self.get_or_allocate_page(PageIndex(index)); + page.bytes.borrow_mut().copy_from_slice(src); + + let end = index + .checked_add(1) + .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) + .ok_or(Error::OffsetOverflow)?; + + self.len.set(cmp::max(self.len.get(), end)); + + Ok(()) + } + + fn get_page(&self, index: PageIndex) -> Option> { + self.pages.borrow().get(&index).cloned() + } + + fn get_or_allocate_page(&self, index: PageIndex) -> Rc { + let mut pages = self.pages.borrow_mut(); + Rc::clone(pages.entry(index).or_insert_with(|| Rc::new(Page::zeroed()))) + } +} diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs new file mode 100644 index 00000000000..670b68d7aa3 --- /dev/null +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -0,0 +1,261 @@ +use alloc::{ + boxed::Box, + collections::{BTreeMap, VecDeque}, + rc::Rc, +}; +use core::{ + cell::RefCell, + future::{poll_fn, Future}, + pin::Pin, + result::Result, + task::Poll, +}; +use futures_channel::oneshot; + +use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; + +mod fs; +mod op; + +pub use crate::io::SECTOR_SIZE; +pub use fs::File; + +#[derive(Debug)] +pub enum Error { + FileNotFound { path: Box }, + FileAlreadyExists { path: Box }, + ShortWrite { expected: usize, written: usize }, + UnexpectedEof { expected: usize, read: usize }, + Fs(fs::Error), +} + +impl From for Error { + fn from(e: fs::Error) -> Self { + Self::Fs(e) + } +} + +#[derive(Default)] +pub struct SimulatorIO { + inner: Rc>, +} + +impl SimulatorIO { + pub fn tick(&self) { + self.inner.borrow_mut().tick(); + } + + fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + self.inner.borrow_mut().submit(op(tx)); + rx + } + + // TODO: The sim runtime should be advancing I/O. Until it does, `tick()` + // whenever a result future is polled and returns pending. + async fn wait_for(&self, mut rx: oneshot::Receiver) -> Result { + poll_fn(|cx| match Pin::new(&mut rx).poll(cx) { + Poll::Ready(result) => Poll::Ready(result), + Poll::Pending => { + self.tick(); + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await + } + + async fn submit_and_wait( + &self, + op: impl FnOnce(oneshot::Sender) -> Box, + ) -> Result { + let rx = self.submit(op); + self.wait_for(rx).await + } +} + +impl SpacetimeIO for SimulatorIO { + type Fd = fs::File; + type Error = Error; + + async fn open_file(&self, path: &str) -> Result { + self.submit_and_wait(|tx| op::open_file(path, tx)) + .await + .expect("`open_file` future cancelled") + } + + async fn create_file(&self, path: &str, len: u64) -> Result { + self.submit_and_wait(|tx| op::create_file(path, len, tx)) + .await + .expect("`create_file` future cancelled") + } + + async fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let () = B::ASSERT_VALID_LAYOUT; + + if !offset.is_multiple_of(SECTOR_SIZE as _) { + self.submit_and_wait(|tx| { + op::ready( + Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }), + tx, + ) + }) + .await + .expect("`write_all_at` future cancelled") + } else { + let (tx, rx) = oneshot::channel(); + for op in op::write_at(fd, buf, offset, tx) { + self.inner.borrow_mut().submit(op); + } + self.wait_for(rx).await.expect("`write_all_at` future cancelled") + } + } + + async fn read_exact_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let () = B::ASSERT_VALID_LAYOUT; + + if !offset.is_multiple_of(SECTOR_SIZE as _) { + self.submit_and_wait(|tx| { + op::ready( + Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }), + tx, + ) + }) + .await + .expect("`read_exact_at` future cancelled") + } else { + let (tx, rx) = oneshot::channel(); + for op in op::read_at(fd, buf, offset, tx) { + self.inner.borrow_mut().submit(op); + } + self.wait_for(rx).await.expect("`read_exact_at` future cancelled") + } + } + + async fn fsync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { + Ok(()) + } + + async fn fdatasync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { + Ok(()) + } + + async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { + let len = self + .submit_and_wait(|tx| op::get_len(fd.clone(), tx)) + .await + .expect("`get_len` future cancelled")?; + self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) + .await + .expect("`set_len` future cancelled") + } +} + +#[derive(Default)] +struct SimulatorIOInner { + files: BTreeMap, fs::File>, + submissions: VecDeque>, + completions: VecDeque>, +} + +impl SimulatorIOInner { + fn tick(&mut self) { + if let Some(sqe) = self.submissions.pop_front() { + sqe.execute(&mut self.files, &mut self.completions); + } + if let Some(cqe) = self.completions.pop_front() { + cqe.complete(); + } + } + + fn submit(&mut self, op: Box) { + self.submissions.push_back(op); + } +} + +trait Submission { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ); +} + +trait Completion { + fn complete(self: Box); +} + +#[cfg(test)] +mod tests { + use crate::sim::Runtime; + + use super::*; + + #[test] + fn create_file() { + let mut rt = Runtime::new(1); + let io = SimulatorIO::default(); + + let fd = rt + .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) + .unwrap(); + assert_eq!(fd.len(), 2 * SECTOR_SIZE as u64); + } + + #[repr(C, align(4096))] + struct Buf([u8; 2 * SECTOR_SIZE]); + + impl AlignedBytes for Buf { + fn as_bytes(&self) -> &[u8] { + &self.0 + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 + } + + fn from_bytes(b: &[u8]) -> Self { + assert_eq!(b.len(), 2 * SECTOR_SIZE); + let mut buf = [0; 2 * SECTOR_SIZE]; + buf.copy_from_slice(b); + Self(buf) + } + } + + #[test] + fn write_read_roundtrip() { + let mut rt = Runtime::new(1); + let io = SimulatorIO::default(); + + let fd = rt + .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) + .unwrap(); + let mut buf = rt + .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) + .map_err(|ErrorWith { error, .. }| error) + .unwrap(); + buf.0.fill(0); + let buf = rt + .block_on(io.read_exact_at(fd, buf, 0)) + .map_err(|ErrorWith { error, .. }| error) + .unwrap(); + + assert!(buf.0.iter().all(|&b| b == 22)); + } +} diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs new file mode 100644 index 00000000000..921129d2536 --- /dev/null +++ b/crates/runtime-core/src/sim/io/op.rs @@ -0,0 +1,322 @@ +use alloc::{ + boxed::Box, + collections::{btree_map, BTreeMap, VecDeque}, + rc::Rc, +}; +use core::cell::RefCell; +use futures_channel::oneshot; + +use super::{fs, Completion, Error, Submission}; +use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; + +pub type WriteAtResult = Result>; +pub type ReadAtResult = Result>; + +pub fn write_at( + fd: fs::File, + buf: B, + offset: u64, + notify: oneshot::Sender>, +) -> impl Iterator> { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Rc::new(RefCell::new(PagedOpState { + buf: Some(buf), + notify: Some(notify), + remaining: page_count, + first_error: None, + })); + + (0..page_count).map(move |buf_page| { + let op = WritePage { + fd: fd.clone(), + file_page: first_page + buf_page, + buf_page, + state: state.clone(), + }; + + Box::new(op) as Box + }) +} + +pub fn read_at( + fd: fs::File, + buf: B, + offset: u64, + notify: oneshot::Sender>, +) -> impl Iterator> { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Rc::new(RefCell::new(PagedOpState { + buf: Some(buf), + notify: Some(notify), + remaining: page_count, + first_error: None, + })); + + (0..page_count).map(move |buf_page| { + let op = ReadPage { + fd: fd.clone(), + file_page: first_page + buf_page, + buf_page, + state: state.clone(), + }; + + Box::new(op) as Box + }) +} + +pub fn open_file(path: &str, notify: oneshot::Sender>) -> Box { + Box::new(OpenFile { + path: path.into(), + notify, + }) +} + +pub fn create_file(path: &str, len: u64, notify: oneshot::Sender>) -> Box { + Box::new(CreateFile { + path: path.into(), + len, + notify, + }) +} + +pub fn get_len(fd: fs::File, notify: oneshot::Sender>) -> Box { + Box::new(GetLen { fd, notify }) +} + +pub fn set_len(fd: fs::File, len: u64, notify: oneshot::Sender>) -> Box { + Box::new(SetLen { fd, len, notify }) +} + +struct GenericCompletion { + result: T, + notify: oneshot::Sender, +} + +fn completion(result: T, notify: oneshot::Sender) -> Box { + Box::new(GenericCompletion { result, notify }) +} + +impl Completion for GenericCompletion { + fn complete(self: Box) { + let Self { result, notify } = *self; + let _ = notify.send(result); + } +} + +struct Ready(Box); + +impl Submission for Ready { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self(completion) = *self; + completions.push_back(completion); + } +} + +pub fn ready(result: T, notify: oneshot::Sender) -> Box { + Box::new(Ready(completion(result, notify))) +} + +struct OpenFile { + path: Box, + notify: oneshot::Sender>, +} + +impl Submission for OpenFile { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { path, notify } = *self; + let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); + completions.push_back(completion(result, notify)); + } +} + +struct CreateFile { + path: Box, + len: u64, + notify: oneshot::Sender>, +} + +impl Submission for CreateFile { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { path, len, notify } = *self; + let result = (|| { + let file = match files.entry(path.clone()) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(_) => Err(Error::FileAlreadyExists { path }), + }?; + file.set_len(len)?; + Ok(file) + })(); + completions.push_back(completion(result, notify)); + } +} + +struct GetLen { + fd: fs::File, + notify: oneshot::Sender>, +} + +impl Submission for GetLen { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { fd, notify } = *self; + let result = Ok(fd.len()); + completions.push_back(completion(result, notify)); + } +} + +struct SetLen { + fd: fs::File, + len: u64, + notify: oneshot::Sender>, +} + +impl Submission for SetLen { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { fd, len, notify } = *self; + let result = fd.set_len(len).map_err(Error::from); + completions.push_back(completion(result, notify)); + } +} + +struct PagedOpState { + buf: Option, + notify: Option>>>, + remaining: usize, + first_error: Option, +} + +fn complete_page_op( + state: &Rc>>, + result: Result<(), fs::Error>, + completions: &mut VecDeque>, +) { + let complete = { + let mut state = state.borrow_mut(); + if let Err(e) = result + && state.first_error.is_none() + { + state.first_error.replace(e.into()); + } + assert!(state.remaining > 0); + state.remaining -= 1; + + state.remaining == 0 + }; + + if complete { + completions.push_back(Box::new(WriteCompletion { state: state.clone() })); + } +} + +struct WriteCompletion { + state: Rc>>, +} + +impl Completion for WriteCompletion { + fn complete(self: Box) { + let (notify, result) = { + let mut state = self.state.borrow_mut(); + + assert_eq!(state.remaining, 0); + + let buf = state.buf.take().expect("write completed more than once"); + let notify = state.notify.take().expect("write completed more than once"); + + let result = match state.first_error.take() { + None => Ok(buf), + Some(error) => Err(ErrorWith { error, with: buf }), + }; + + (notify, result) + }; + + let _ = notify.send(result); + } +} + +struct WritePage { + fd: fs::File, + file_page: usize, + buf_page: usize, + state: Rc>>, +} + +impl Submission for WritePage { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { + fd, + file_page, + buf_page, + state, + } = *self; + + let result = { + let state_ref = state.borrow(); + let buf = state_ref.buf.as_ref().expect("buffer went away"); + + let start = buf_page * SECTOR_SIZE; + let end = start + SECTOR_SIZE; + fd.write_page(&buf.as_bytes()[start..end], file_page as _) + }; + complete_page_op(&state, result, completions); + } +} + +struct ReadPage { + fd: fs::File, + file_page: usize, + buf_page: usize, + state: Rc>>, +} + +impl Submission for ReadPage { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { + fd, + file_page, + buf_page, + state, + } = *self; + + let result = { + let mut state_ref = state.borrow_mut(); + let buf = state_ref.buf.as_mut().expect("buffer went away"); + + let start = buf_page * SECTOR_SIZE; + let end = start + SECTOR_SIZE; + fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) + }; + complete_page_op(&state, result, completions); + } +} diff --git a/crates/runtime-core/src/sim/mod.rs b/crates/runtime-core/src/sim/mod.rs index e2c231828a1..1a5a53a29bf 100644 --- a/crates/runtime-core/src/sim/mod.rs +++ b/crates/runtime-core/src/sim/mod.rs @@ -1,5 +1,6 @@ pub mod buggify; mod executor; +pub mod io; mod rng; pub mod time; diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index c8affea0f48..2b2cffc5317 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -13,6 +13,10 @@ workspace = true tokio.workspace = true spacetimedb-runtime-core = { workspace = true, optional = true } libc = { version = "0.2", optional = true } +static_assertions = "1.1" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [dev-dependencies] futures.workspace = true diff --git a/crates/runtime/src/io.rs b/crates/runtime/src/io.rs new file mode 100644 index 00000000000..f7c24fe029f --- /dev/null +++ b/crates/runtime/src/io.rs @@ -0,0 +1,2 @@ +mod tokio; +pub use tokio::TokioIO as Tokio; diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs new file mode 100644 index 00000000000..eb948ed7e1c --- /dev/null +++ b/crates/runtime/src/io/tokio.rs @@ -0,0 +1,163 @@ +use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; + +#[cfg(unix)] +use std::os::unix::fs::FileExt as _; +#[cfg(windows)] +use std::os::windows::fs::FileExt as _; + +use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use static_assertions::assert_not_impl_any; +use tokio::fs::OpenOptions; +use tokio::{runtime, task::spawn_blocking}; + +/// Implementation of [SpacetimeIO] that runs on a tokio runtime. +pub struct TokioIO { + // TODO: Should this be [runtime::Runtime]? + rt: runtime::Handle, + // Ensure I/O stays on a single thread. + _not_send: PhantomData>, +} + +impl TokioIO { + pub fn new(rt: runtime::Handle) -> Self { + Self { + rt, + _not_send: PhantomData, + } + } +} + +assert_not_impl_any!(TokioIO: Send); + +impl SpacetimeIO for TokioIO { + // NOTE: This operates on a [std::fs::File] handle instead of + // [tokio::fs::File] because `pwrite`/`pread`-style APIs are not available + // from tokio proper. As a consequence, operations on an open `Fd` use + // [spawn_blocking]. This is what [tokio::fs::File] does internally, while + // here we can avoid some locking. + type Fd = Arc; + type Error = io::Error; + + async fn open_file(&self, path: &str) -> Result { + let _rt = self.rt.enter(); + + let mut open_options = tokio::fs::File::options(); + open_options.read(true).write(true); + let file = open_with_direct_io(open_options, path).await?; + + Ok(Arc::new(file.into_std().await)) + } + + async fn create_file(&self, path: &str, len: u64) -> Result { + let _rt = self.rt.enter(); + + let mut open_options = tokio::fs::File::options(); + open_options.read(true).write(true).create_new(true); + let file = open_with_direct_io(open_options, path).await?; + file.set_len(len).await?; + + Ok(Arc::new(file.into_std().await)) + } + + async fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let _rt = self.rt.enter(); + asyncify(move || { + #[cfg(unix)] + let res = fd.write_all_at(buf.as_bytes(), offset); + #[cfg(windows)] + let res = fd.seek_write(buf.as_bytes(), offset); + + match res { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + } + }) + .await + } + + async fn read_exact_at( + &self, + fd: Self::Fd, + mut buf: B, + offset: u64, + ) -> Result> { + let _rt = self.rt.enter(); + asyncify(move || { + #[cfg(unix)] + let res = fd.read_exact_at(buf.as_bytes_mut(), offset); + #[cfg(windows)] + let res = fd.seek_read(buf.as_bytes_mut(), offset); + + match res { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + } + }) + .await + } + + async fn fsync(&self, fd: Self::Fd) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || fd.sync_all()).await + } + + async fn fdatasync(&self, fd: Self::Fd) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || fd.sync_data()).await + } + + async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || { + let len = fd.metadata()?.len(); + fd.set_len(len + additional)?; + + Ok(()) + }) + .await + } +} + +async fn asyncify(f: F) -> R +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + spawn_blocking(f).await.unwrap_or_else(|e| match e.try_into_panic() { + Ok(panic_payload) => std::panic::resume_unwind(panic_payload), + // A cancellation should not be possible, because we await the task. + Err(e) => panic!("unexpected error joining blocking task: {e}"), + }) +} + +#[cfg(all(unix, not(target_os = "macos")))] +async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { + options.custom_flags(libc::O_DIRECT).open(path).await +} + +#[cfg(target_os = "macos")] +async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { + let file = options.open(path).await?; + asyncify(move || { + let res = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; + if res == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(file) + } + }) + .await +} + +#[cfg(target_os = "windows")] +async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { + options + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) + .open(path) + .await +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index c6192e1b738..48400676009 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -53,6 +53,8 @@ pub enum Handle { Simulation(sim::Handle), } +pub mod io; + pub struct JoinHandle { inner: JoinHandleInner, } From 11dc4c7197ebc2bd2a948af0bb3fc6c5b4295b1a Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 4 Aug 2026 15:00:51 +0200 Subject: [PATCH 02/19] Use SimulatorIO as the "I/O driver" in the executor Entails making it Send + Sync, which may or may not be what we want. --- crates/runtime-core/src/sim/executor/mod.rs | 26 +++++- crates/runtime-core/src/sim/io/fs.rs | 50 +++++------ crates/runtime-core/src/sim/io/mod.rs | 92 ++++++++++----------- crates/runtime-core/src/sim/io/op.rs | 41 +++++---- 4 files changed, 114 insertions(+), 95 deletions(-) diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index fbb7f7c0cf2..eee88b4a88a 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -10,6 +10,8 @@ use core::{ use spin::Mutex; +use crate::sim::io::SimulatorIO; + use super::{time::TimeHandle, Rng}; mod task; @@ -21,11 +23,12 @@ type Runnable = async_task::Runnable; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RuntimeConfig { pub seed: u64, + pub enable_io: bool, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed } + Self { seed, enable_io: false } } } @@ -145,6 +148,12 @@ impl Runtime { } } + // TODO: This is a stopgap to allow submission of I/O tasks. We probably + // want the user-facing API to hide this. + pub fn io(&self) -> &Option { + &self.executor.io + } + /// Drive a top-level future to completion on the simulation executor. /// /// While the future runs, spawned tasks share the same deterministic @@ -360,6 +369,7 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, + io: Option, } impl Executor { @@ -375,6 +385,7 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), + io: config.enable_io.then(SimulatorIO::default), } } @@ -499,6 +510,10 @@ impl Executor { }; } + if self.run_pending_io() { + continue; + } + if self.time.wake_next_timer() { continue; } @@ -527,6 +542,15 @@ impl Executor { } } + fn run_pending_io(&self) -> bool { + // TODO: Inject faults (reorder, delay, drop, ..) when buggify is enabled. + // Also, should this run more than one queue entry? + match &self.io { + Some(io) => io.tick(), + None => false, + } + } + /// Look up the record for a node, panicking if the node is unknown. fn node_record(&self, node: NodeId) -> Arc { self.nodes diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 9c540f5e1b3..bdc64b87657 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,7 +1,7 @@ -use alloc::{collections::BTreeMap, rc::Rc}; +use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cell::{Cell, RefCell}, cmp, + sync::atomic::{AtomicU64, Ordering}, }; pub const PAGE_SIZE: usize = 4096; @@ -27,13 +27,13 @@ impl PageIndex { } struct Page { - bytes: RefCell<[u8; PAGE_SIZE]>, + bytes: spin::Mutex<[u8; PAGE_SIZE]>, } impl Page { fn zeroed() -> Self { Self { - bytes: RefCell::new([0; PAGE_SIZE]), + bytes: spin::Mutex::new([0; PAGE_SIZE]), } } } @@ -47,25 +47,25 @@ impl Page { /// or written. Writing a page is atomic. #[derive(Clone)] pub struct File { - pages: RefCell>>, - len: Cell, + pages: Arc>>>, + len: Arc, } impl File { - pub(super) const fn new() -> Self { + pub(super) fn new() -> Self { Self { - pages: RefCell::new(BTreeMap::new()), - len: Cell::new(0), + pages: Arc::new(spin::Mutex::new(BTreeMap::new())), + len: Arc::new(AtomicU64::new(0)), } } - pub(super) const fn len(&self) -> u64 { - self.len.get() + pub(super) fn len(&self) -> u64 { + self.len.load(Ordering::Relaxed) } #[allow(unused)] - pub(super) const fn is_empty(&self) -> bool { - self.len.get() == 0 + pub(super) fn is_empty(&self) -> bool { + self.len() == 0 } /// Change the file length. @@ -80,7 +80,7 @@ impl File { if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } - let old_len = self.len.get(); + let old_len = self.len(); match new_len.cmp(&old_len) { Equal => {} @@ -92,13 +92,13 @@ impl File { self.get_or_allocate_page(PageIndex(index)); } - self.len.set(new_len); + self.len.store(new_len, Ordering::Relaxed); } Less => { - self.len.set(new_len); + self.len.store(new_len, Ordering::Relaxed); let first_removed = PageIndex::from_offset(new_len); - let removed = self.pages.borrow_mut().split_off(&first_removed); + let removed = self.pages.lock().split_off(&first_removed); drop(removed); } } @@ -114,7 +114,7 @@ impl File { match self.get_page(PageIndex(index)) { Some(page) => { - dst.copy_from_slice(&*page.bytes.borrow()); + dst.copy_from_slice(&*page.bytes.lock()); } None => { dst.fill(0); @@ -131,24 +131,24 @@ impl File { } let page = self.get_or_allocate_page(PageIndex(index)); - page.bytes.borrow_mut().copy_from_slice(src); + page.bytes.lock().copy_from_slice(src); let end = index .checked_add(1) .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) .ok_or(Error::OffsetOverflow)?; - self.len.set(cmp::max(self.len.get(), end)); + self.len.fetch_max(end, Ordering::Relaxed); Ok(()) } - fn get_page(&self, index: PageIndex) -> Option> { - self.pages.borrow().get(&index).cloned() + fn get_page(&self, index: PageIndex) -> Option> { + self.pages.lock().get(&index).cloned() } - fn get_or_allocate_page(&self, index: PageIndex) -> Rc { - let mut pages = self.pages.borrow_mut(); - Rc::clone(pages.entry(index).or_insert_with(|| Rc::new(Page::zeroed()))) + fn get_or_allocate_page(&self, index: PageIndex) -> Arc { + let mut pages = self.pages.lock(); + Arc::clone(pages.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) } } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 670b68d7aa3..6f8f9af8569 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -1,15 +1,9 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, - rc::Rc, -}; -use core::{ - cell::RefCell, - future::{poll_fn, Future}, - pin::Pin, - result::Result, - task::Poll, + sync::Arc, }; +use core::result::Result; use futures_channel::oneshot; use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; @@ -35,42 +29,23 @@ impl From for Error { } } -#[derive(Default)] +#[derive(Clone, Default)] pub struct SimulatorIO { - inner: Rc>, + inner: Arc>, } impl SimulatorIO { - pub fn tick(&self) { - self.inner.borrow_mut().tick(); - } - - fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> oneshot::Receiver { - let (tx, rx) = oneshot::channel(); - self.inner.borrow_mut().submit(op(tx)); - rx - } - - // TODO: The sim runtime should be advancing I/O. Until it does, `tick()` - // whenever a result future is polled and returns pending. - async fn wait_for(&self, mut rx: oneshot::Receiver) -> Result { - poll_fn(|cx| match Pin::new(&mut rx).poll(cx) { - Poll::Ready(result) => Poll::Ready(result), - Poll::Pending => { - self.tick(); - cx.waker().wake_by_ref(); - Poll::Pending - } - }) - .await + pub fn tick(&self) -> bool { + self.inner.lock().tick() } async fn submit_and_wait( &self, op: impl FnOnce(oneshot::Sender) -> Box, ) -> Result { - let rx = self.submit(op); - self.wait_for(rx).await + let (tx, rx) = oneshot::channel(); + self.inner.lock().submit(op(tx)); + rx.await } } @@ -90,7 +65,7 @@ impl SpacetimeIO for SimulatorIO { .expect("`create_file` future cancelled") } - async fn write_all_at( + async fn write_all_at( &self, fd: Self::Fd, buf: B, @@ -113,13 +88,13 @@ impl SpacetimeIO for SimulatorIO { } else { let (tx, rx) = oneshot::channel(); for op in op::write_at(fd, buf, offset, tx) { - self.inner.borrow_mut().submit(op); + self.inner.lock().submit(op); } - self.wait_for(rx).await.expect("`write_all_at` future cancelled") + rx.await.expect("`write_all_at` future cancelled") } } - async fn read_exact_at( + async fn read_exact_at( &self, fd: Self::Fd, buf: B, @@ -142,9 +117,9 @@ impl SpacetimeIO for SimulatorIO { } else { let (tx, rx) = oneshot::channel(); for op in op::read_at(fd, buf, offset, tx) { - self.inner.borrow_mut().submit(op); + self.inner.lock().submit(op); } - self.wait_for(rx).await.expect("`read_exact_at` future cancelled") + rx.await.expect("`read_exact_at` future cancelled") } } @@ -175,13 +150,28 @@ struct SimulatorIOInner { } impl SimulatorIOInner { - fn tick(&mut self) { + // TODO: Allow runtime to inject faults via: + // + // - pick random entries from the submission queue + // - drop queue entries + // - delay `execute` (somehow) + // - delay `complete` + // - make a submission fail without performing its effect + // - execute an arbitrary number of (random) SQEs + // - complete an arbitrary number of CQEs + + fn tick(&mut self) -> bool { + let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { sqe.execute(&mut self.files, &mut self.completions); + progress = true; } if let Some(cqe) = self.completions.pop_front() { cqe.complete(); + progress = true; } + + progress } fn submit(&mut self, op: Box) { @@ -189,7 +179,7 @@ impl SimulatorIOInner { } } -trait Submission { +trait Submission: Send { fn execute( self: Box, files: &mut BTreeMap, fs::File>, @@ -197,20 +187,23 @@ trait Submission { ); } -trait Completion { +trait Completion: Send { fn complete(self: Box); } #[cfg(test)] mod tests { - use crate::sim::Runtime; + use crate::sim::{Runtime, RuntimeConfig}; use super::*; #[test] fn create_file() { - let mut rt = Runtime::new(1); - let io = SimulatorIO::default(); + let mut rt = Runtime::with_config(RuntimeConfig { + enable_io: true, + ..<_>::default() + }); + let io = rt.io().clone().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) @@ -240,8 +233,11 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::new(1); - let io = SimulatorIO::default(); + let mut rt = Runtime::with_config(RuntimeConfig { + enable_io: true, + ..<_>::default() + }); + let io = rt.io().clone().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 921129d2536..963f26dc522 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,9 +1,8 @@ use alloc::{ boxed::Box, collections::{btree_map, BTreeMap, VecDeque}, - rc::Rc, + sync::Arc, }; -use core::cell::RefCell; use futures_channel::oneshot; use super::{fs, Completion, Error, Submission}; @@ -12,7 +11,7 @@ use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; pub type WriteAtResult = Result>; pub type ReadAtResult = Result>; -pub fn write_at( +pub fn write_at( fd: fs::File, buf: B, offset: u64, @@ -21,7 +20,7 @@ pub fn write_at( let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; - let state = Rc::new(RefCell::new(PagedOpState { + let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), notify: Some(notify), remaining: page_count, @@ -40,7 +39,7 @@ pub fn write_at( }) } -pub fn read_at( +pub fn read_at( fd: fs::File, buf: B, offset: u64, @@ -49,7 +48,7 @@ pub fn read_at( let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; - let state = Rc::new(RefCell::new(PagedOpState { + let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), notify: Some(notify), remaining: page_count, @@ -96,11 +95,11 @@ struct GenericCompletion { notify: oneshot::Sender, } -fn completion(result: T, notify: oneshot::Sender) -> Box { +fn completion(result: T, notify: oneshot::Sender) -> Box { Box::new(GenericCompletion { result, notify }) } -impl Completion for GenericCompletion { +impl Completion for GenericCompletion { fn complete(self: Box) { let Self { result, notify } = *self; let _ = notify.send(result); @@ -120,7 +119,7 @@ impl Submission for Ready { } } -pub fn ready(result: T, notify: oneshot::Sender) -> Box { +pub fn ready(result: T, notify: oneshot::Sender) -> Box { Box::new(Ready(completion(result, notify))) } @@ -208,13 +207,13 @@ struct PagedOpState { first_error: Option, } -fn complete_page_op( - state: &Rc>>, +fn complete_page_op( + state: &Arc>>, result: Result<(), fs::Error>, completions: &mut VecDeque>, ) { let complete = { - let mut state = state.borrow_mut(); + let mut state = state.lock(); if let Err(e) = result && state.first_error.is_none() { @@ -232,13 +231,13 @@ fn complete_page_op( } struct WriteCompletion { - state: Rc>>, + state: Arc>>, } -impl Completion for WriteCompletion { +impl Completion for WriteCompletion { fn complete(self: Box) { let (notify, result) = { - let mut state = self.state.borrow_mut(); + let mut state = self.state.lock(); assert_eq!(state.remaining, 0); @@ -261,10 +260,10 @@ struct WritePage { fd: fs::File, file_page: usize, buf_page: usize, - state: Rc>>, + state: Arc>>, } -impl Submission for WritePage { +impl Submission for WritePage { fn execute( self: Box, _files: &mut BTreeMap, fs::File>, @@ -278,7 +277,7 @@ impl Submission for WritePage { } = *self; let result = { - let state_ref = state.borrow(); + let state_ref = state.lock(); let buf = state_ref.buf.as_ref().expect("buffer went away"); let start = buf_page * SECTOR_SIZE; @@ -293,10 +292,10 @@ struct ReadPage { fd: fs::File, file_page: usize, buf_page: usize, - state: Rc>>, + state: Arc>>, } -impl Submission for ReadPage { +impl Submission for ReadPage { fn execute( self: Box, _files: &mut BTreeMap, fs::File>, @@ -310,7 +309,7 @@ impl Submission for ReadPage { } = *self; let result = { - let mut state_ref = state.borrow_mut(); + let mut state_ref = state.lock(); let buf = state_ref.buf.as_mut().expect("buffer went away"); let start = buf_page * SECTOR_SIZE; From f47352a360197dd259fb13c7903b502885c05a7b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 4 Aug 2026 16:29:02 +0200 Subject: [PATCH 03/19] Make it clearer that we're clearing --- crates/runtime-core/src/sim/io/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 6f8f9af8569..91ac80527b4 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -214,6 +214,12 @@ mod tests { #[repr(C, align(4096))] struct Buf([u8; 2 * SECTOR_SIZE]); + impl Buf { + fn clear(&mut self) { + self.0.fill(0); + } + } + impl AlignedBytes for Buf { fn as_bytes(&self) -> &[u8] { &self.0 @@ -246,7 +252,7 @@ mod tests { .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) .map_err(|ErrorWith { error, .. }| error) .unwrap(); - buf.0.fill(0); + buf.clear(); let buf = rt .block_on(io.read_exact_at(fd, buf, 0)) .map_err(|ErrorWith { error, .. }| error) From 243930b95fb2503e7852c7a4f457cb0d787f5ecb Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 6 Aug 2026 18:27:58 +0200 Subject: [PATCH 04/19] Expose ways for the runtime to inject failures. --- crates/runtime-core/src/sim/io/mod.rs | 78 +++++++-- crates/runtime-core/src/sim/io/op.rs | 234 ++++++++++++++++---------- 2 files changed, 205 insertions(+), 107 deletions(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 91ac80527b4..e50c77f558a 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -2,14 +2,19 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, sync::Arc, + vec::Vec, }; -use core::result::Result; +use core::{ops::RangeBounds, result::Result}; use futures_channel::oneshot; -use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use crate::{ + io::{AlignedBytes, ErrorWith, SpacetimeIO}, + sim::Rng, +}; mod fs; -mod op; +pub mod op; +use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; @@ -31,14 +36,46 @@ impl From for Error { #[derive(Clone, Default)] pub struct SimulatorIO { + // TODO: We make `SimulatorIO` `Send + Sync` for now, because + // [crate::sim::executor::Handle] is just `Arc`. This means that a + // future carrying a handle can't be `spawn`ed, because spawning requires + // the future to be `Send`. + // + // We should fix this at some point, so below can become `Rc>`. inner: Arc>, } impl SimulatorIO { + /// Run the submission at the front of the queue (if any), and complete the + /// completion at the front of the queue (if any). pub fn tick(&self) -> bool { self.inner.lock().tick() } + /// Execute `sqe`. + pub fn execute(&self, sqe: Box) { + self.inner.lock().execute(sqe); + } + + /// Remove and return the submission at the fron of the queue, if any. + pub fn next_submission(&self) -> Option> { + self.inner.lock().next() + } + + /// Remove and return a random submission, or `None` if the queue is empty. + pub fn random_submission(&self, rng: &Rng) -> Option> { + self.inner.lock().next_random(rng) + } + + /// Remove `range` from the completion queue. + pub fn completions(&self, range: impl RangeBounds) -> impl Iterator> { + self.inner + .lock() + .drain_completions(range) + .collect::>() + .into_iter() + } + async fn submit_and_wait( &self, op: impl FnOnce(oneshot::Sender) -> Box, @@ -163,7 +200,9 @@ impl SimulatorIOInner { fn tick(&mut self) -> bool { let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { - sqe.execute(&mut self.files, &mut self.completions); + if let Some(cqe) = sqe.execute(&mut self.files) { + self.completions.push_back(cqe); + } progress = true; } if let Some(cqe) = self.completions.pop_front() { @@ -174,21 +213,28 @@ impl SimulatorIOInner { progress } - fn submit(&mut self, op: Box) { - self.submissions.push_back(op); + fn execute(&mut self, sqe: Box) { + if let Some(cqe) = sqe.execute(&mut self.files) { + self.completions.push_back(cqe); + } } -} -trait Submission: Send { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ); -} + fn next(&mut self) -> Option> { + self.submissions.pop_front() + } + + fn next_random(&mut self, rng: &Rng) -> Option> { + let i = rng.next_u64() % self.submissions.len() as u64; + self.submissions.remove(i as usize) + } + + fn drain_completions(&mut self, range: impl RangeBounds) -> impl Iterator> { + self.completions.drain(range) + } -trait Completion: Send { - fn complete(self: Box); + fn submit(&mut self, op: Box) { + self.submissions.push_back(op); + } } #[cfg(test)] diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 963f26dc522..a4e95d71fb6 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,28 +1,60 @@ +use core::any::Any; + use alloc::{ boxed::Box, - collections::{btree_map, BTreeMap, VecDeque}, + collections::{btree_map, BTreeMap}, sync::Arc, }; use futures_channel::oneshot; -use super::{fs, Completion, Error, Submission}; +use super::{fs, Error}; use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; +/// An operation that can be submitted to the [super::SimulatorIO] driver. +pub trait Submission: Send + Any { + /// Run the operations with mutable access to the currently registered + /// [fs::File]s. + /// + /// If the operation is done, a [Completion] is returned in a `Some`. + /// `None` may be returned if: + /// + /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. + /// - The submission is a [Noop]. + /// + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; +} + +/// An object containing the result of executing a [Submission], as well as a +/// handle to resolve a future waiting on the outcome of the operation. +pub trait Completion: Send { + /// Resolve the future waiting on the outcome of the operation. + fn complete(self: Box); +} + +/// A channel to resolve a future waiting on the outcome of a submitted +/// operation. +pub type OnComplete = oneshot::Sender; + pub type WriteAtResult = Result>; -pub type ReadAtResult = Result>; +/// Write the contents of `buf` to `fd` at `offset`. +/// +/// This operation is split into multiple writes to individual pages. The +/// `on_complete` future resolves only after all page writes completed. +/// +/// Ownership of `buf` is transferred back when the operation completes. pub fn write_at( fd: fs::File, buf: B, offset: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, ) -> impl Iterator> { let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), - notify: Some(notify), + on_complete: Some(on_complete), remaining: page_count, first_error: None, })); @@ -39,18 +71,27 @@ pub fn write_at( }) } +pub type ReadAtResult = Result>; + +/// Fill `buf` by reading from `fd` at `offset`. +/// +/// This operation is split into multple reads from the individual pages needed +/// to fill `buf`. The `on_complete` future resolves only after all page reads +/// completed. +/// +/// Ownership of `buf` is transferred back when the operation completes. pub fn read_at( fd: fs::File, buf: B, offset: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, ) -> impl Iterator> { let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), - notify: Some(notify), + on_complete: Some(on_complete), remaining: page_count, first_error: None, })); @@ -67,92 +108,107 @@ pub fn read_at( }) } -pub fn open_file(path: &str, notify: oneshot::Sender>) -> Box { +/// Open file at `path`. +pub fn open_file(path: &str, on_complete: OnComplete>) -> Box { Box::new(OpenFile { path: path.into(), - notify, + on_complete, }) } -pub fn create_file(path: &str, len: u64, notify: oneshot::Sender>) -> Box { +/// Create a new file at `path` and allocate `len` space for it. +pub fn create_file(path: &str, len: u64, on_complete: OnComplete>) -> Box { Box::new(CreateFile { path: path.into(), len, - notify, + on_complete, }) } -pub fn get_len(fd: fs::File, notify: oneshot::Sender>) -> Box { - Box::new(GetLen { fd, notify }) +/// Get the length of the file `fd`. +pub fn get_len(fd: fs::File, on_complete: OnComplete>) -> Box { + Box::new(GetLen { fd, on_complete }) } -pub fn set_len(fd: fs::File, len: u64, notify: oneshot::Sender>) -> Box { - Box::new(SetLen { fd, len, notify }) +/// Set the length of the file `fd`. +pub fn set_len(fd: fs::File, len: u64, on_complete: OnComplete>) -> Box { + Box::new(SetLen { fd, len, on_complete }) } struct GenericCompletion { result: T, - notify: oneshot::Sender, + on_complete: OnComplete, } -fn completion(result: T, notify: oneshot::Sender) -> Box { - Box::new(GenericCompletion { result, notify }) +fn completion(result: T, on_complete: OnComplete) -> Box { + Box::new(GenericCompletion { result, on_complete }) } impl Completion for GenericCompletion { fn complete(self: Box) { - let Self { result, notify } = *self; - let _ = notify.send(result); + let Self { + result, on_complete, .. + } = *self; + let _ = on_complete.send(result); } } -struct Ready(Box); +/// [Submission] created by [noop]. +pub(crate) struct Noop; + +impl Submission for Noop { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + None + } +} + +/// An operation that does nothing. +/// +/// Note that no completion is associated with a noop, but the submission still +/// occupies a slot in the submission queue. +pub fn noop() -> Box { + Box::new(Noop) +} + +/// [Submission] created by [ready]. +pub(crate) struct Ready(Box); impl Submission for Ready { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self(completion) = *self; - completions.push_back(completion); + Some(completion) } } -pub fn ready(result: T, notify: oneshot::Sender) -> Box { - Box::new(Ready(completion(result, notify))) +/// An operation that is already complete with `result`. +pub fn ready(result: T, on_complete: OnComplete) -> Box { + Box::new(Ready(completion(result, on_complete))) } -struct OpenFile { +/// [Submission] created by [open_file]. +pub(crate) struct OpenFile { path: Box, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for OpenFile { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { path, notify } = *self; + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { path, on_complete } = *self; let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct CreateFile { +/// [Submission] created by [create_file]. +pub(crate) struct CreateFile { path: Box, len: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for CreateFile { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { path, len, notify } = *self; + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { path, len, on_complete } = *self; let result = (|| { let file = match files.entry(path.clone()) { btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), @@ -161,48 +217,42 @@ impl Submission for CreateFile { file.set_len(len)?; Ok(file) })(); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct GetLen { +/// [Submission] created by [get_len]. +pub(crate) struct GetLen { fd: fs::File, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for GetLen { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { fd, notify } = *self; + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + let Self { fd, on_complete } = *self; let result = Ok(fd.len()); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct SetLen { +/// [Submission] created by [set_len]. +pub(crate) struct SetLen { fd: fs::File, len: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for SetLen { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { fd, len, notify } = *self; + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + let Self { fd, len, on_complete } = *self; let result = fd.set_len(len).map_err(Error::from); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } struct PagedOpState { buf: Option, - notify: Option>>>, + on_complete: Option>>>, remaining: usize, first_error: Option, } @@ -210,8 +260,7 @@ struct PagedOpState { fn complete_page_op( state: &Arc>>, result: Result<(), fs::Error>, - completions: &mut VecDeque>, -) { +) -> Option>> { let complete = { let mut state = state.lock(); if let Err(e) = result @@ -225,38 +274,36 @@ fn complete_page_op( state.remaining == 0 }; - if complete { - completions.push_back(Box::new(WriteCompletion { state: state.clone() })); - } + complete.then(|| Box::new(PageOpCompletion { state: state.clone() })) } -struct WriteCompletion { +struct PageOpCompletion { state: Arc>>, } -impl Completion for WriteCompletion { +impl Completion for PageOpCompletion { fn complete(self: Box) { - let (notify, result) = { + let (on_complete, result) = { let mut state = self.state.lock(); assert_eq!(state.remaining, 0); let buf = state.buf.take().expect("write completed more than once"); - let notify = state.notify.take().expect("write completed more than once"); + let on_complete = state.on_complete.take().expect("write completed more than once"); let result = match state.first_error.take() { None => Ok(buf), Some(error) => Err(ErrorWith { error, with: buf }), }; - (notify, result) + (on_complete, result) }; - let _ = notify.send(result); + let _ = on_complete.send(result); } } -struct WritePage { +pub(crate) struct WritePage { fd: fs::File, file_page: usize, buf_page: usize, @@ -264,11 +311,7 @@ struct WritePage { } impl Submission for WritePage { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, file_page, @@ -284,11 +327,11 @@ impl Submission for WritePage { let end = start + SECTOR_SIZE; fd.write_page(&buf.as_bytes()[start..end], file_page as _) }; - complete_page_op(&state, result, completions); + complete_page_op(&state, result).map(|c| c as Box) } } -struct ReadPage { +pub(crate) struct ReadPage { fd: fs::File, file_page: usize, buf_page: usize, @@ -296,11 +339,7 @@ struct ReadPage { } impl Submission for ReadPage { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, file_page, @@ -316,6 +355,19 @@ impl Submission for ReadPage { let end = start + SECTOR_SIZE; fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) }; - complete_page_op(&state, result, completions); + complete_page_op(&state, result).map(|c| c as Box) + } +} + +#[cfg(test)] +mod tests { + use core::any::Any; + + use super::*; + + #[test] + fn downcast() { + let sqe: Box = noop(); + sqe.downcast::().unwrap(); } } From 40faf30984511ed7963b5d3a66963390ae5c490b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 14:01:11 +0200 Subject: [PATCH 05/19] Encapsulate SimulatorIO in an I/O "driver" that can inject failures. --- crates/runtime-core/src/sim/executor/io.rs | 101 ++++++++++++++++++++ crates/runtime-core/src/sim/executor/mod.rs | 45 +++++---- crates/runtime-core/src/sim/io/fs.rs | 6 +- crates/runtime-core/src/sim/io/mod.rs | 92 +++++++++++------- crates/runtime-core/src/sim/io/op.rs | 75 ++++++++++++++- 5 files changed, 262 insertions(+), 57 deletions(-) create mode 100644 crates/runtime-core/src/sim/executor/io.rs diff --git a/crates/runtime-core/src/sim/executor/io.rs b/crates/runtime-core/src/sim/executor/io.rs new file mode 100644 index 00000000000..ca1831978bf --- /dev/null +++ b/crates/runtime-core/src/sim/executor/io.rs @@ -0,0 +1,101 @@ +use crate::sim::{io::SimulatorIO, Rng}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Config { + /// The max number of submissions to run per [Driver::tick]. + pub max_submissions_per_tick: usize, + /// The max number of completions to finish per [Driver::tick]. + pub max_completions_per_tick: usize, + /// Submission reordering probability. + /// + /// Describes the probability by which to select the next submission queue + /// entry randomly, as opposed to the oldest entry in the queue. + pub prob_reorder_submissions: f64, + /// Completion reordering probability. + /// + /// Describes the probability by which to select the next completion queue + /// entry randomly, as opposed to the oldest entry in the queue. + pub prob_reorder_completions: f64, + /// Probability by which to skip one submission queue entry. + /// + /// If skipped, the entry still counts towards `max_submissions_per_tick`. + pub prob_skip: f64, + /// Probability by which to cancel a submission queue entry. + /// + /// [crate::sim::io::op::Submission::cancel()] is called on the entry, which + /// may generate a completion. + pub prob_cancel: f64, +} + +impl Default for Config { + fn default() -> Self { + Self { + max_submissions_per_tick: 1, + max_completions_per_tick: 1, + prob_reorder_submissions: 0.0, + prob_reorder_completions: 0.0, + prob_skip: 0.0, + prob_cancel: 0.0, + } + } +} + +pub struct Driver { + io: SimulatorIO, + config: Config, +} + +impl Driver { + pub fn new(config: Config) -> Self { + Self { + io: <_>::default(), + config, + } + } + + /// Advance the I/O simulator according the [Config]. + /// + /// Returns `true` if progress has been made, or there are pending entries + /// in either the submission or completion queue. + pub fn tick(&self, rng: &Rng) -> bool { + let mut progress = false; + for _ in 0..self.config.max_submissions_per_tick { + if !rng.buggify_with_prob(self.config.prob_skip) { + let sqe = if rng.buggify_with_prob(self.config.prob_reorder_submissions) { + self.io.random_submission(rng) + } else { + self.io.next_submission() + }; + + if let Some(sqe) = sqe { + if rng.buggify_with_prob(self.config.prob_cancel) { + sqe.cancel(); + } else { + self.io.execute(sqe); + } + progress = true; + } + } + } + + for _ in 0..self.config.max_completions_per_tick { + let cqe = if rng.buggify_with_prob(self.config.prob_reorder_completions) { + self.io.random_completion(rng) + } else { + self.io.next_completion() + }; + + if let Some(cqe) = cqe { + cqe.complete(); + progress = true + } + } + + progress |= self.io.pending(); + progress + } + + pub fn io(&self) -> &SimulatorIO { + &self.io + } +} diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index eee88b4a88a..1913329b2e2 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -14,21 +14,34 @@ use crate::sim::io::SimulatorIO; use super::{time::TimeHandle, Rng}; +mod io; + mod task; use task::Abortable; pub use task::{AbortHandle, JoinError, JoinHandle}; type Runnable = async_task::Runnable; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct RuntimeConfig { pub seed: u64, - pub enable_io: bool, + pub io: Option, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed, enable_io: false } + Self { seed, io: None } + } + + pub fn enable_io(self) -> Self { + Self { + io: Some(self.io.unwrap_or_default()), + ..self + } + } + + pub fn with_io_config(self, io: Option) -> Self { + Self { io, ..self } } } @@ -150,8 +163,8 @@ impl Runtime { // TODO: This is a stopgap to allow submission of I/O tasks. We probably // want the user-facing API to hide this. - pub fn io(&self) -> &Option { - &self.executor.io + pub fn io(&self) -> Option<&SimulatorIO> { + self.executor.io.as_ref().map(|driver| driver.io()) } /// Drive a top-level future to completion on the simulation executor. @@ -369,7 +382,7 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, - io: Option, + io: Option, } impl Executor { @@ -385,7 +398,7 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), - io: config.enable_io.then(SimulatorIO::default), + io: config.io.map(io::Driver::new), } } @@ -502,6 +515,7 @@ impl Executor { loop { self.run_all_ready(); + let pending_io = self.drive_io(); if task.is_finished() { let waker = Waker::noop(); return match Pin::new(&mut task).poll(&mut Context::from_waker(waker)) { @@ -510,11 +524,7 @@ impl Executor { }; } - if self.run_pending_io() { - continue; - } - - if self.time.wake_next_timer() { + if self.time.wake_next_timer() || pending_io { continue; } @@ -542,12 +552,11 @@ impl Executor { } } - fn run_pending_io(&self) -> bool { - // TODO: Inject faults (reorder, delay, drop, ..) when buggify is enabled. - // Also, should this run more than one queue entry? - match &self.io { - Some(io) => io.tick(), - None => false, + fn drive_io(&self) -> bool { + if let Some(io) = &self.io { + io.tick(&self.rng) + } else { + false } } diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index bdc64b87657..908c8b7ba83 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -74,7 +74,7 @@ impl File { /// /// Extending allocates pages eagerly as needed. Shrinking drops all pages /// at or beyond the new EOF. - pub fn set_len(&self, new_len: u64) -> Result<()> { + pub(super) fn set_len(&self, new_len: u64) -> Result<()> { use cmp::Ordering::*; if !new_len.is_multiple_of(PAGE_SIZE_U64) { @@ -107,7 +107,7 @@ impl File { } /// Read one complete page. - pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { + pub(super) fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { if dst.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } @@ -125,7 +125,7 @@ impl File { } /// Write one complete page. - pub fn write_page(&self, src: &[u8], index: u64) -> Result<()> { + pub(super) fn write_page(&self, src: &[u8], index: u64) -> Result<()> { if src.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index e50c77f558a..c9a69fcb8ba 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -2,9 +2,8 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, sync::Arc, - vec::Vec, }; -use core::{ops::RangeBounds, result::Result}; +use core::{num::NonZeroUsize, result::Result}; use futures_channel::oneshot; use crate::{ @@ -21,11 +20,23 @@ pub use fs::File; #[derive(Debug)] pub enum Error { - FileNotFound { path: Box }, - FileAlreadyExists { path: Box }, - ShortWrite { expected: usize, written: usize }, - UnexpectedEof { expected: usize, read: usize }, + FileNotFound { + path: Box, + }, + FileAlreadyExists { + path: Box, + }, + ShortWrite { + expected: usize, + written: usize, + }, + UnexpectedEof { + expected: usize, + read: usize, + }, Fs(fs::Error), + /// Injected by the I/O driver. + Cancelled, } impl From for Error { @@ -46,6 +57,23 @@ pub struct SimulatorIO { } impl SimulatorIO { + /// Returns `true` if there are entries in either the submission or + /// completion queues. + pub fn pending(&self) -> bool { + let inner = self.inner.lock(); + inner.submissions.len() + inner.completions.len() > 0 + } + + /// Number of entries in the submission queue. + pub fn pending_submissions(&self) -> usize { + self.inner.lock().submissions.len() + } + + /// Number of entries in the completion queue. + pub fn pending_completions(&self) -> usize { + self.inner.lock().completions.len() + } + /// Run the submission at the front of the queue (if any), and complete the /// completion at the front of the queue (if any). pub fn tick(&self) -> bool { @@ -57,23 +85,24 @@ impl SimulatorIO { self.inner.lock().execute(sqe); } - /// Remove and return the submission at the fron of the queue, if any. + /// Remove and return the submission at the front of the queue, if any. pub fn next_submission(&self) -> Option> { - self.inner.lock().next() + self.inner.lock().next_submission() } /// Remove and return a random submission, or `None` if the queue is empty. pub fn random_submission(&self, rng: &Rng) -> Option> { - self.inner.lock().next_random(rng) + self.inner.lock().random_submission(rng) + } + + /// Remove and return the completion at the front of the queue, if any. + pub fn next_completion(&self) -> Option> { + self.inner.lock().next_completion() } - /// Remove `range` from the completion queue. - pub fn completions(&self, range: impl RangeBounds) -> impl Iterator> { - self.inner - .lock() - .drain_completions(range) - .collect::>() - .into_iter() + /// Remove and return a random completion, or `None` if the queue is empty. + pub fn random_completion(&self, rng: &Rng) -> Option> { + self.inner.lock().random_completion(rng) } async fn submit_and_wait( @@ -219,17 +248,22 @@ impl SimulatorIOInner { } } - fn next(&mut self) -> Option> { + fn next_submission(&mut self) -> Option> { self.submissions.pop_front() } - fn next_random(&mut self, rng: &Rng) -> Option> { - let i = rng.next_u64() % self.submissions.len() as u64; - self.submissions.remove(i as usize) + fn random_submission(&mut self, rng: &Rng) -> Option> { + let len = NonZeroUsize::new(self.submissions.len())?; + self.submissions.remove(rng.index(len.get())) + } + + fn next_completion(&mut self) -> Option> { + self.completions.pop_front() } - fn drain_completions(&mut self, range: impl RangeBounds) -> impl Iterator> { - self.completions.drain(range) + fn random_completion(&mut self, rng: &Rng) -> Option> { + let len = NonZeroUsize::new(self.completions.len())?; + self.completions.remove(rng.index(len.get())) } fn submit(&mut self, op: Box) { @@ -245,11 +279,8 @@ mod tests { #[test] fn create_file() { - let mut rt = Runtime::with_config(RuntimeConfig { - enable_io: true, - ..<_>::default() - }); - let io = rt.io().clone().unwrap(); + let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); + let io = rt.io().cloned().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) @@ -285,11 +316,8 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::with_config(RuntimeConfig { - enable_io: true, - ..<_>::default() - }); - let io = rt.io().clone().unwrap(); + let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); + let io = rt.io().cloned().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index a4e95d71fb6..3c2b1f90086 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -22,6 +22,16 @@ pub trait Submission: Send + Any { /// - The submission is a [Noop]. /// fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; + + /// Cancel the operation instead of executing it. + /// + /// This will generate a [Completion] with the result [Error::Cancelled], + /// unless: + /// + /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. + /// - The submission is a [Noop]. + /// + fn cancel(self: Box) -> Option>; } /// An object containing the result of executing a [Submission], as well as a @@ -160,6 +170,10 @@ impl Submission for Noop { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { None } + + fn cancel(self: Box) -> Option> { + None + } } /// An operation that does nothing. @@ -178,6 +192,11 @@ impl Submission for Ready { let Self(completion) = *self; Some(completion) } + + fn cancel(self: Box) -> Option> { + let Self(completion) = *self; + Some(completion) + } } /// An operation that is already complete with `result`. @@ -197,6 +216,11 @@ impl Submission for OpenFile { let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { path: _, on_complete } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [create_file]. @@ -219,6 +243,15 @@ impl Submission for CreateFile { })(); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { + path: _, + len: _, + on_complete, + } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [get_len]. @@ -233,6 +266,11 @@ impl Submission for GetLen { let result = Ok(fd.len()); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { fd: _, on_complete } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [set_len]. @@ -248,6 +286,15 @@ impl Submission for SetLen { let result = fd.set_len(len).map_err(Error::from); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + len: _, + on_complete, + } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } struct PagedOpState { @@ -259,14 +306,14 @@ struct PagedOpState { fn complete_page_op( state: &Arc>>, - result: Result<(), fs::Error>, + result: Result<(), Error>, ) -> Option>> { let complete = { let mut state = state.lock(); if let Err(e) = result && state.first_error.is_none() { - state.first_error.replace(e.into()); + state.first_error.replace(e); } assert!(state.remaining > 0); state.remaining -= 1; @@ -327,7 +374,17 @@ impl Submission for WritePage { let end = start + SECTOR_SIZE; fd.write_page(&buf.as_bytes()[start..end], file_page as _) }; - complete_page_op(&state, result).map(|c| c as Box) + complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) + } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + file_page: _, + buf_page: _, + state, + } = *self; + complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) } } @@ -355,7 +412,17 @@ impl Submission for ReadPage { let end = start + SECTOR_SIZE; fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) }; - complete_page_op(&state, result).map(|c| c as Box) + complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) + } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + file_page: _, + buf_page: _, + state, + } = *self; + complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) } } From c60c637ce2bc18f1a760f04a23d9f510e269fcec Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 14:08:19 +0200 Subject: [PATCH 06/19] Remove TODO --- crates/runtime-core/src/sim/io/mod.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index c9a69fcb8ba..6ea3a2eaa1d 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -216,16 +216,6 @@ struct SimulatorIOInner { } impl SimulatorIOInner { - // TODO: Allow runtime to inject faults via: - // - // - pick random entries from the submission queue - // - drop queue entries - // - delay `execute` (somehow) - // - delay `complete` - // - make a submission fail without performing its effect - // - execute an arbitrary number of (random) SQEs - // - complete an arbitrary number of CQEs - fn tick(&mut self) -> bool { let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { From 51b1f6e51d8110b26a72849af27e8b192af39630 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 15:30:52 +0200 Subject: [PATCH 07/19] Fix optional dependencies --- crates/runtime/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 2b2cffc5317..d23741ce139 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -11,10 +11,12 @@ workspace = true [dependencies] tokio.workspace = true -spacetimedb-runtime-core = { workspace = true, optional = true } -libc = { version = "0.2", optional = true } +spacetimedb-runtime-core = { workspace = true } static_assertions = "1.1" +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } @@ -22,4 +24,4 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } futures.workspace = true [features] -simulation = ["dep:spacetimedb-runtime-core", "spacetimedb-runtime-core/sim", "dep:libc"] +simulation = ["spacetimedb-runtime-core/sim"] From 36ca1f48c7ed13003b2b8958044e4c05a979538c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 17:02:48 +0200 Subject: [PATCH 08/19] Fix windows --- crates/runtime/src/io/tokio.rs | 74 ++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index eb948ed7e1c..7907a3f7cab 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -66,16 +66,9 @@ impl SpacetimeIO for TokioIO { offset: u64, ) -> Result> { let _rt = self.rt.enter(); - asyncify(move || { - #[cfg(unix)] - let res = fd.write_all_at(buf.as_bytes(), offset); - #[cfg(windows)] - let res = fd.seek_write(buf.as_bytes(), offset); - - match res { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - } + asyncify(move || match write_all_at(&fd, buf.as_bytes(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), }) .await } @@ -87,16 +80,9 @@ impl SpacetimeIO for TokioIO { offset: u64, ) -> Result> { let _rt = self.rt.enter(); - asyncify(move || { - #[cfg(unix)] - let res = fd.read_exact_at(buf.as_bytes_mut(), offset); - #[cfg(windows)] - let res = fd.seek_read(buf.as_bytes_mut(), offset); - - match res { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - } + asyncify(move || match read_exact_at(&fd, buf.as_bytes_mut(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), }) .await } @@ -154,10 +140,56 @@ async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result io::Result { options .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) .open(path) .await } + +#[cfg(unix)] +#[inline] +fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { + fd.read_exact_at(buf, offset) +} + +#[cfg(windows)] +fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match file.seek_read(buf, offset) { + Ok(0) => return Err(ErrorKind::UnexpectedEof.into()), + Ok(n) => { + offset += n as u64; + buf = &mut buf[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + + Ok(()) +} + +#[cfg(unix)] +#[inline] +fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { + fd.write_all_at(buf, offset) +} + +#[cfg(windows)] +fn write_all_at(fd: &std::fd::File, buf: &[u8], offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match file.seek_write(buf, offset) { + Ok(0) => return Err(ErrorKind::WriteZero.into()), + Ok(n) => { + offset += n as u64; + buf = &buf[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + + Ok(()) +} From 45362078c27de7f67d5b91fa9748f907aa03fc69 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Sat, 8 Aug 2026 12:24:23 +0200 Subject: [PATCH 09/19] Fix fix windows --- crates/runtime/src/io/tokio.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 7907a3f7cab..dcc77dbc5b4 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -141,7 +141,7 @@ async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result io::Result { +async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { options .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) .open(path) @@ -155,15 +155,15 @@ fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result< } #[cfg(windows)] -fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], mut offset: u64) -> io::Result<()> { +fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { while !buf.is_empty() { - match file.seek_read(buf, offset) { - Ok(0) => return Err(ErrorKind::UnexpectedEof.into()), + match fd.seek_read(buf, offset) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), Ok(n) => { offset += n as u64; buf = &mut buf[n..]; } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} Err(e) => return Err(e), } } @@ -178,15 +178,15 @@ fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { } #[cfg(windows)] -fn write_all_at(fd: &std::fd::File, buf: &[u8], offset: u64) -> io::Result<()> { +fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { while !buf.is_empty() { - match file.seek_write(buf, offset) { - Ok(0) => return Err(ErrorKind::WriteZero.into()), + match fd.seek_write(buf, offset) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), Ok(n) => { offset += n as u64; buf = &buf[n..]; } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} Err(e) => return Err(e), } } From d4da8686eb198815dea6b8bde875e8b5bf2fdb86 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Sat, 8 Aug 2026 13:35:46 +0200 Subject: [PATCH 10/19] Add a length function for files --- crates/runtime-core/src/io/mod.rs | 5 +++++ crates/runtime-core/src/sim/io/mod.rs | 9 +++++---- crates/runtime/src/io/tokio.rs | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index b4ff3744916..24a27579a2d 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -145,4 +145,9 @@ pub trait SpacetimeIO { /// Allocate `additional` bytes for the file `fd`. fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; + + /// Determine the length of the file `fd`. + /// + /// This should not depend on `fsync`, i.e. `statx`. See `std::io::Seek::stream_len`. + fn length(&self, fd: Self::Fd) -> Self::Completion>; } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 6ea3a2eaa1d..bb8ed855106 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -198,14 +198,15 @@ impl SpacetimeIO for SimulatorIO { } async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let len = self - .submit_and_wait(|tx| op::get_len(fd.clone(), tx)) - .await - .expect("`get_len` future cancelled")?; + let len = self.length(fd.clone()).await?; self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) .await .expect("`set_len` future cancelled") } + + fn length(&self, fd: Self::Fd) -> Self::Completion> { + self.submit(op::get_len(fd)) + } } #[derive(Default)] diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index dcc77dbc5b4..2dcf2ab1ab6 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -1,3 +1,8 @@ +use std::io::{Seek, SeekFrom}; +use std::panic; +use std::path::PathBuf; +use std::pin::Pin; +use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; #[cfg(unix)] @@ -107,6 +112,15 @@ impl SpacetimeIO for TokioIO { }) .await } + + fn length(&self, fd: Self::Fd) -> Self::Completion> { + self.rt + .spawn_blocking(move || { + let mut fd = fd.try_clone()?; + file_length(&mut fd) + }) + .into() + } } async fn asyncify(f: F) -> R From be161e4c19be1b28f1f3601e9a2364743d40d479 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 18 Aug 2026 16:36:07 +0200 Subject: [PATCH 11/19] Fix editor fuckup, satisfy error trait bound --- Cargo.lock | 1 + crates/runtime-core/Cargo.toml | 1 + crates/runtime-core/src/sim/io/fs.rs | 5 ++++- crates/runtime-core/src/sim/io/mod.rs | 26 +++++++++++--------------- crates/runtime/src/io/tokio.rs | 1 - 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0289ef17a31..c38b884a6ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8345,6 +8345,7 @@ dependencies = [ "async-task", "futures-channel", "spin", + "thiserror 2.0.17", "zerocopy", ] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 6ac037162dd..6b644b6ab92 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -17,4 +17,5 @@ sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] async-task = { version = "4.4", default-features = false, optional = true } futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } +thiserror = { version = "2.0", default-features = false } zerocopy = "0.8" diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 908c8b7ba83..efa3d9ab755 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -7,10 +7,13 @@ use core::{ pub const PAGE_SIZE: usize = 4096; const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] pub enum Error { + #[error("unaligned offset")] UnalignedOffset, + #[error("unaligned buffer")] UnalignedBuffer, + #[error("offset overflow")] OffsetOverflow, } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index bb8ed855106..c81cb5ca270 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -18,24 +18,20 @@ use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { - FileNotFound { - path: Box, - }, - FileAlreadyExists { - path: Box, - }, - ShortWrite { - expected: usize, - written: usize, - }, - UnexpectedEof { - expected: usize, - read: usize, - }, + #[error("file not found")] + FileNotFound { path: Box }, + #[error("file already exists")] + FileAlreadyExists { path: Box }, + #[error("failed to write expected number of bytes")] + ShortWrite { expected: usize, written: usize }, + #[error("unexpected eof")] + UnexpectedEof { expected: usize, read: usize }, + #[error(transparent)] Fs(fs::Error), /// Injected by the I/O driver. + #[error("operation cancelled")] Cancelled, } diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 2dcf2ab1ab6..d48e6623a62 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -17,7 +17,6 @@ use tokio::{runtime, task::spawn_blocking}; /// Implementation of [SpacetimeIO] that runs on a tokio runtime. pub struct TokioIO { - // TODO: Should this be [runtime::Runtime]? rt: runtime::Handle, // Ensure I/O stays on a single thread. _not_send: PhantomData>, From d498da1ae99a684ad4a5bee2ec3f1d05102ca0dd Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 21 Aug 2026 10:43:14 +0200 Subject: [PATCH 12/19] Unify completion future --- crates/runtime-core/src/io/mod.rs | 14 +- crates/runtime-core/src/sim/io/mod.rs | 141 +++++++------ crates/runtime-core/src/sim/io/op.rs | 265 +++++++++++++++++------- crates/runtime/src/io/tokio.rs | 280 +++++++++++++++----------- 4 files changed, 433 insertions(+), 267 deletions(-) diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index 24a27579a2d..7b54b1dd39b 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -101,12 +101,12 @@ pub trait SpacetimeIO { type Error; /// Open the file at `path`. - fn open_file(&self, path: &str) -> impl Future>; + fn open_file(&self, path: &str) -> Self::Completion>; /// Create the file at `path` and allocate `len` bytes. /// /// Returns an error if the file already exists. - fn create_file(&self, path: &str, len: u64) -> impl Future>; + fn create_file(&self, path: &str, len: u64) -> Self::Completion>; /// Write `buf` to `fd` at `offset`. /// @@ -120,7 +120,7 @@ pub trait SpacetimeIO { fd: Self::Fd, buf: B, offset: u64, - ) -> impl Future>>; + ) -> Self::Completion>>; /// Read `size_of::()` bytes from `fd` at `offset` and interpret them at /// type `B`. @@ -136,15 +136,15 @@ pub trait SpacetimeIO { fd: Self::Fd, buf: B, offset: u64, - ) -> impl Future>>; + ) -> Self::Completion>>; /// Call `fsync(2)` on `fd`. - fn fsync(&self, fd: Self::Fd) -> impl Future>; + fn fsync(&self, fd: Self::Fd) -> Self::Completion>; /// Call `fdatasync(2)` on `fd`. - fn fdatasync(&self, fd: Self::Fd) -> impl Future>; + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion>; /// Allocate `additional` bytes for the file `fd`. - fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; + fn reserve(&self, fd: Self::Fd, additional: u64) -> Self::Completion>; /// Determine the length of the file `fd`. /// diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index c81cb5ca270..a86d4b0834a 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -3,7 +3,12 @@ use alloc::{ collections::{BTreeMap, VecDeque}, sync::Arc, }; -use core::{num::NonZeroUsize, result::Result}; +use core::{ + num::NonZeroUsize, + pin::Pin, + result::Result, + task::{Context, Poll}, +}; use futures_channel::oneshot; use crate::{ @@ -13,7 +18,6 @@ use crate::{ mod fs; pub mod op; -use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; @@ -77,127 +81,118 @@ impl SimulatorIO { } /// Execute `sqe`. - pub fn execute(&self, sqe: Box) { + pub fn execute(&self, sqe: Box) { self.inner.lock().execute(sqe); } /// Remove and return the submission at the front of the queue, if any. - pub fn next_submission(&self) -> Option> { + pub fn next_submission(&self) -> Option> { self.inner.lock().next_submission() } /// Remove and return a random submission, or `None` if the queue is empty. - pub fn random_submission(&self, rng: &Rng) -> Option> { + pub fn random_submission(&self, rng: &Rng) -> Option> { self.inner.lock().random_submission(rng) } /// Remove and return the completion at the front of the queue, if any. - pub fn next_completion(&self) -> Option> { + pub fn next_completion(&self) -> Option> { self.inner.lock().next_completion() } /// Remove and return a random completion, or `None` if the queue is empty. - pub fn random_completion(&self, rng: &Rng) -> Option> { + pub fn random_completion(&self, rng: &Rng) -> Option> { self.inner.lock().random_completion(rng) } - async fn submit_and_wait( - &self, - op: impl FnOnce(oneshot::Sender) -> Box, - ) -> Result { + fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> Completion { let (tx, rx) = oneshot::channel(); self.inner.lock().submit(op(tx)); - rx.await + Completion(rx) + } + + fn submit_all>>( + &self, + ops: impl FnOnce(oneshot::Sender) -> I, + ) -> Completion { + let (tx, rx) = oneshot::channel(); + let mut inner = self.inner.lock(); + ops(tx).for_each(|op| inner.submit(op)); + Completion(rx) + } +} + +#[must_use = "completions must be polled to completion"] +pub struct Completion(oneshot::Receiver); + +impl Future for Completion { + type Output = T; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.as_mut().0).poll(cx).map(Result::unwrap) } } impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; + type Completion = Completion; - async fn open_file(&self, path: &str) -> Result { - self.submit_and_wait(|tx| op::open_file(path, tx)) - .await - .expect("`open_file` future cancelled") + fn open_file(&self, path: &str) -> Self::Completion> { + self.submit(op::open_file(path)) } - async fn create_file(&self, path: &str, len: u64) -> Result { - self.submit_and_wait(|tx| op::create_file(path, len, tx)) - .await - .expect("`create_file` future cancelled") + fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + self.submit(op::create_file(path, len)) } - async fn write_all_at( + fn write_all_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { + ) -> Self::Completion>> { let () = B::ASSERT_VALID_LAYOUT; if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit_and_wait(|tx| { - op::ready( - Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }), - tx, - ) - }) - .await - .expect("`write_all_at` future cancelled") + self.submit(op::ready(Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }))) } else { - let (tx, rx) = oneshot::channel(); - for op in op::write_at(fd, buf, offset, tx) { - self.inner.lock().submit(op); - } - rx.await.expect("`write_all_at` future cancelled") + self.submit_all(op::write_at(fd, buf, offset)) } } - async fn read_exact_at( + fn read_exact_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { + ) -> Self::Completion>> { let () = B::ASSERT_VALID_LAYOUT; if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit_and_wait(|tx| { - op::ready( - Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }), - tx, - ) - }) - .await - .expect("`read_exact_at` future cancelled") + self.submit(op::ready(Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }))) } else { - let (tx, rx) = oneshot::channel(); - for op in op::read_at(fd, buf, offset, tx) { - self.inner.lock().submit(op); - } - rx.await.expect("`read_exact_at` future cancelled") + self.submit_all(op::read_at(fd, buf, offset)) } } - async fn fsync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { - Ok(()) + fn fsync(&self, _fd: Self::Fd) -> Self::Completion> { + self.submit(op::ready(Ok(()))) } - async fn fdatasync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { - Ok(()) + fn fdatasync(&self, _fd: Self::Fd) -> Self::Completion> { + self.submit(op::ready(Ok(()))) } - async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let len = self.length(fd.clone()).await?; - self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) - .await - .expect("`set_len` future cancelled") + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { + assert!(total >= fd.len()); + self.submit(op::set_len(fd, total)) } fn length(&self, fd: Self::Fd) -> Self::Completion> { @@ -208,8 +203,8 @@ impl SpacetimeIO for SimulatorIO { #[derive(Default)] struct SimulatorIOInner { files: BTreeMap, fs::File>, - submissions: VecDeque>, - completions: VecDeque>, + submissions: VecDeque>, + completions: VecDeque>, } impl SimulatorIOInner { @@ -229,31 +224,31 @@ impl SimulatorIOInner { progress } - fn execute(&mut self, sqe: Box) { + fn execute(&mut self, sqe: Box) { if let Some(cqe) = sqe.execute(&mut self.files) { self.completions.push_back(cqe); } } - fn next_submission(&mut self) -> Option> { + fn next_submission(&mut self) -> Option> { self.submissions.pop_front() } - fn random_submission(&mut self, rng: &Rng) -> Option> { + fn random_submission(&mut self, rng: &Rng) -> Option> { let len = NonZeroUsize::new(self.submissions.len())?; self.submissions.remove(rng.index(len.get())) } - fn next_completion(&mut self) -> Option> { + fn next_completion(&mut self) -> Option> { self.completions.pop_front() } - fn random_completion(&mut self, rng: &Rng) -> Option> { + fn random_completion(&mut self, rng: &Rng) -> Option> { let len = NonZeroUsize::new(self.completions.len())?; self.completions.remove(rng.index(len.get())) } - fn submit(&mut self, op: Box) { + fn submit(&mut self, op: Box) { self.submissions.push_back(op); } } diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 3c2b1f90086..0e855313819 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,4 +1,4 @@ -use core::any::Any; +use core::{any::Any, iter::Scan, ops::Range}; use alloc::{ boxed::Box, @@ -37,6 +37,7 @@ pub trait Submission: Send + Any { /// An object containing the result of executing a [Submission], as well as a /// handle to resolve a future waiting on the outcome of the operation. pub trait Completion: Send { + fn success(&self) -> bool; /// Resolve the future waiting on the outcome of the operation. fn complete(self: Box); } @@ -45,6 +46,10 @@ pub trait Completion: Send { /// operation. pub type OnComplete = oneshot::Sender; +pub type ScanState = (fs::File, usize, Arc>>); +pub type PageWrites = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; +pub type PageReads = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; + pub type WriteAtResult = Result>; /// Write the contents of `buf` to `fd` at `offset`. @@ -57,28 +62,29 @@ pub fn write_at( fd: fs::File, buf: B, offset: u64, - on_complete: OnComplete>, -) -> impl Iterator> { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).map(move |buf_page| { - let op = WritePage { - fd: fd.clone(), - file_page: first_page + buf_page, - buf_page, - state: state.clone(), - }; +) -> impl FnOnce(OnComplete>) -> PageWrites { + move |on_complete| { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Arc::new(spin::Mutex::new(PagedOpState { + buf: Some(buf), + on_complete: Some(on_complete), + remaining: page_count, + first_error: None, + })); + + (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { + let op = WritePage { + fd: fd.clone(), + file_page: *first_page + buf_page, + buf_page, + state: state.clone(), + }; - Box::new(op) as Box - }) + Some(Box::new(op)) + }) + } } pub type ReadAtResult = Result>; @@ -94,67 +100,81 @@ pub fn read_at( fd: fs::File, buf: B, offset: u64, - on_complete: OnComplete>, -) -> impl Iterator> { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).map(move |buf_page| { - let op = ReadPage { - fd: fd.clone(), - file_page: first_page + buf_page, - buf_page, - state: state.clone(), - }; +) -> impl FnOnce(OnComplete>) -> PageReads { + move |on_complete| { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Arc::new(spin::Mutex::new(PagedOpState { + buf: Some(buf), + on_complete: Some(on_complete), + remaining: page_count, + first_error: None, + })); + + (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { + let op = ReadPage { + fd: fd.clone(), + file_page: *first_page + buf_page, + buf_page, + state: state.clone(), + }; - Box::new(op) as Box - }) + Some(Box::new(op)) + }) + } } /// Open file at `path`. -pub fn open_file(path: &str, on_complete: OnComplete>) -> Box { - Box::new(OpenFile { - path: path.into(), - on_complete, - }) +pub fn open_file(path: &str) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| { + Box::new(OpenFile { + path: path.into(), + on_complete, + }) + } } /// Create a new file at `path` and allocate `len` space for it. -pub fn create_file(path: &str, len: u64, on_complete: OnComplete>) -> Box { - Box::new(CreateFile { - path: path.into(), - len, - on_complete, - }) +pub fn create_file(path: &str, len: u64) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| { + Box::new(CreateFile { + path: path.into(), + len, + on_complete, + }) + } } /// Get the length of the file `fd`. -pub fn get_len(fd: fs::File, on_complete: OnComplete>) -> Box { - Box::new(GetLen { fd, on_complete }) +pub fn get_len(fd: fs::File) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| Box::new(GetLen { fd, on_complete }) } /// Set the length of the file `fd`. -pub fn set_len(fd: fs::File, len: u64, on_complete: OnComplete>) -> Box { - Box::new(SetLen { fd, len, on_complete }) +pub fn set_len(fd: fs::File, len: u64) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| Box::new(SetLen { fd, len, on_complete }) } struct GenericCompletion { + success: bool, result: T, on_complete: OnComplete, } -fn completion(result: T, on_complete: OnComplete) -> Box { - Box::new(GenericCompletion { result, on_complete }) +fn completion(success: bool, result: T, on_complete: OnComplete) -> Box { + Box::new(GenericCompletion { + success, + result, + on_complete, + }) } impl Completion for GenericCompletion { + fn success(&self) -> bool { + self.success + } + fn complete(self: Box) { let Self { result, on_complete, .. @@ -200,8 +220,107 @@ impl Submission for Ready { } /// An operation that is already complete with `result`. -pub fn ready(result: T, on_complete: OnComplete) -> Box { - Box::new(Ready(completion(result, on_complete))) +pub fn ready(result: T) -> impl FnOnce(OnComplete) -> Box { + move |on_complete| Box::new(Ready(completion(true, result, on_complete))) +} + +/// [Submission] created by [link]. +pub(crate) struct SoftLink { + a: Box, + b: Box, +} + +impl Submission for SoftLink { + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { a, b } = *self; + let result_a = a.execute(files); + let result_b = if result_a.as_ref().is_none_or(|result| result.success()) { + b.execute(files) + } else { + b.cancel() + }; + + Some(Box::new(LinkedCompletion { + a: result_a, + b: result_b, + })) + } + + fn cancel(self: Box) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.cancel(), + b: b.cancel(), + })) + } +} + +/// Link `a` and `b`, such that `b` gets executed after `a`. +/// +/// If `a` fails (i.e. its [Completion::success] returns `false`), `b` is +/// cancelled. +/// +/// Corresponds to io-uring's `IOSQE_IO_LINK` flag. To emulate +/// `IOSQE_IO_HARDLINK`, see [hard_link]. +pub fn link(a: Box, b: Box) -> Box { + Box::new(SoftLink { a, b }) +} + +/// [Submission] created by [hard_link]. +pub(crate) struct HardLink { + a: Box, + b: Box, +} + +impl Submission for HardLink { + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.execute(files), + b: b.execute(files), + })) + } + + fn cancel(self: Box) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.cancel(), + b: b.cancel(), + })) + } +} + +/// Link `a` and `b`, such that `b` gets executed after `a`. +/// +/// Unlike [link], this executes both submissions regardless of the result. It +/// just enforces the ordering constraint that `b` will never execute before +/// `a`. +/// +/// Corresponds to io-uring's `IOSQE_IO_HARDLINK` flag. To emulate +/// `IOSQE_IO_LINK`, see [link]. +pub fn hard_link(a: Box, b: Box) -> Box { + Box::new(HardLink { a, b }) +} + +struct LinkedCompletion { + a: Option>, + b: Option>, +} + +impl Completion for LinkedCompletion { + fn success(&self) -> bool { + self.a.as_ref().is_none_or(|result| result.success()) && self.b.as_ref().is_none_or(|result| result.success()) + } + + fn complete(self: Box) { + let Self { a, b } = *self; + if let Some(a) = a { + a.complete(); + } + if let Some(b) = b { + b.complete(); + } + } } /// [Submission] created by [open_file]. @@ -214,12 +333,12 @@ impl Submission for OpenFile { fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { let Self { path, on_complete } = *self; let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { let Self { path: _, on_complete } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -241,7 +360,7 @@ impl Submission for CreateFile { file.set_len(len)?; Ok(file) })(); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { @@ -250,7 +369,7 @@ impl Submission for CreateFile { len: _, on_complete, } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -264,12 +383,12 @@ impl Submission for GetLen { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, on_complete } = *self; let result = Ok(fd.len()); - Some(completion(result, on_complete)) + Some(completion(true, result, on_complete)) } fn cancel(self: Box) -> Option> { let Self { fd: _, on_complete } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -284,7 +403,7 @@ impl Submission for SetLen { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, len, on_complete } = *self; let result = fd.set_len(len).map_err(Error::from); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { @@ -293,11 +412,11 @@ impl Submission for SetLen { len: _, on_complete, } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } -struct PagedOpState { +pub struct PagedOpState { buf: Option, on_complete: Option>>>, remaining: usize, @@ -329,6 +448,10 @@ struct PageOpCompletion { } impl Completion for PageOpCompletion { + fn success(&self) -> bool { + self.state.lock().first_error.is_none() + } + fn complete(self: Box) { let (on_complete, result) = { let mut state = self.state.lock(); diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index d48e6623a62..1aeb31c0af1 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,15 +5,9 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -#[cfg(unix)] -use std::os::unix::fs::FileExt as _; -#[cfg(windows)] -use std::os::windows::fs::FileExt as _; - use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; use static_assertions::assert_not_impl_any; -use tokio::fs::OpenOptions; -use tokio::{runtime, task::spawn_blocking}; +use tokio::runtime; /// Implementation of [SpacetimeIO] that runs on a tokio runtime. pub struct TokioIO { @@ -33,6 +27,38 @@ impl TokioIO { assert_not_impl_any!(TokioIO: Send); +#[must_use = "completions must be polled to completion"] +pub struct Completion(tokio::task::JoinHandle); + +impl Future for Completion { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match Pin::new(&mut this.0).poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => match result { + Ok(output) => Poll::Ready(output), + Err(error) => { + if error.is_panic() { + panic::resume_unwind(error.into_panic()) + } else if error.is_cancelled() { + panic!("I/O task unexpectedly cancelled"); + } else { + unreachable!("unexpected I/O task error") + } + } + }, + } + } +} + +impl From> for Completion { + fn from(handle: tokio::task::JoinHandle) -> Self { + Self(handle) + } +} + impl SpacetimeIO for TokioIO { // NOTE: This operates on a [std::fs::File] handle instead of // [tokio::fs::File] because `pwrite`/`pread`-style APIs are not available @@ -41,75 +67,77 @@ impl SpacetimeIO for TokioIO { // here we can avoid some locking. type Fd = Arc; type Error = io::Error; + type Completion = Completion; - async fn open_file(&self, path: &str) -> Result { - let _rt = self.rt.enter(); - - let mut open_options = tokio::fs::File::options(); - open_options.read(true).write(true); - let file = open_with_direct_io(open_options, path).await?; - - Ok(Arc::new(file.into_std().await)) + fn open_file(&self, path: &str) -> Self::Completion> { + let path = PathBuf::from(path); + self.rt + .spawn_blocking(move || { + let mut open_options = std::fs::File::options(); + open_options.read(true).write(true); + platform::open_with_direct_io(open_options, path).map(Arc::new) + }) + .into() } - async fn create_file(&self, path: &str, len: u64) -> Result { - let _rt = self.rt.enter(); - - let mut open_options = tokio::fs::File::options(); - open_options.read(true).write(true).create_new(true); - let file = open_with_direct_io(open_options, path).await?; - file.set_len(len).await?; - - Ok(Arc::new(file.into_std().await)) + fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + let path = PathBuf::from(path); + self.rt + .spawn_blocking(move || { + let mut open_options = std::fs::File::options(); + open_options.read(true).write(true).create_new(true); + let file = platform::open_with_direct_io(open_options, path)?; + file.set_len(len)?; + Ok(Arc::new(file)) + }) + .into() } - async fn write_all_at( + fn write_all_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { - let _rt = self.rt.enter(); - asyncify(move || match write_all_at(&fd, buf.as_bytes(), offset) { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - }) - .await + ) -> Self::Completion>> { + self.rt + .spawn_blocking(move || match platform::write_all_at(&fd, buf.as_bytes(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + }) + .into() } - async fn read_exact_at( + fn read_exact_at( &self, fd: Self::Fd, mut buf: B, offset: u64, - ) -> Result> { - let _rt = self.rt.enter(); - asyncify(move || match read_exact_at(&fd, buf.as_bytes_mut(), offset) { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - }) - .await + ) -> Self::Completion>> { + self.rt + .spawn_blocking(move || match platform::read_exact_at(&fd, buf.as_bytes_mut(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + }) + .into() } - async fn fsync(&self, fd: Self::Fd) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || fd.sync_all()).await + fn fsync(&self, fd: Self::Fd) -> Self::Completion> { + self.rt.spawn_blocking(move || fd.sync_all()).into() } - async fn fdatasync(&self, fd: Self::Fd) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || fd.sync_data()).await + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { + self.rt.spawn_blocking(move || fd.sync_data()).into() } - async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || { - let len = fd.metadata()?.len(); - fd.set_len(len + additional)?; - - Ok(()) - }) - .await + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { + self.rt + .spawn_blocking(move || { + let mut fd = fd.try_clone()?; + let len = file_length(&mut fd)?; + assert!(total >= len); + fd.set_len(total) + }) + .into() } fn length(&self, fd: Self::Fd) -> Self::Completion> { @@ -122,87 +150,107 @@ impl SpacetimeIO for TokioIO { } } -async fn asyncify(f: F) -> R -where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, -{ - spawn_blocking(f).await.unwrap_or_else(|e| match e.try_into_panic() { - Ok(panic_payload) => std::panic::resume_unwind(panic_payload), - // A cancellation should not be possible, because we await the task. - Err(e) => panic!("unexpected error joining blocking task: {e}"), - }) +fn file_length(fd: &mut std::fs::File) -> io::Result { + let pos = fd.stream_position()?; + let len = fd.seek(SeekFrom::End(0))?; + + if pos != len { + fd.seek(SeekFrom::Start(pos))?; + } + + Ok(len) } -#[cfg(all(unix, not(target_os = "macos")))] -async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { - options.custom_flags(libc::O_DIRECT).open(path).await +mod platform { + #[cfg(unix)] + pub use super::unix::*; + + #[cfg(windows)] + pub use super::windows::*; } -#[cfg(target_os = "macos")] -async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { - let file = options.open(path).await?; - asyncify(move || { +#[cfg(unix)] +mod unix { + use std::{ + io, + os::unix::fs::{FileExt as _, OpenOptionsExt as _}, + path::Path, + }; + + #[inline] + pub fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { + fd.read_exact_at(buf, offset) + } + + #[inline] + pub fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { + fd.write_all_at(buf, offset) + } + + #[cfg(not(target_os = "macos"))] + pub fn open_with_direct_io(mut options: std::fs::OpenOptions, path: impl AsRef) -> io::Result { + options.custom_flags(libc::O_DIRECT).open(path) + } + + #[cfg(target_os = "macos")] + pub async fn open_with_direct_io( + options: std::fs::OpenOptions, + path: impl AsRef, + ) -> io::Result { + let file = options.open(path)?; let res = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; if res == -1 { Err(io::Error::last_os_error()) } else { Ok(file) } - }) - .await + } } #[cfg(windows)] -async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { - options - .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) - .open(path) - .await -} - -#[cfg(unix)] -#[inline] -fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { - fd.read_exact_at(buf, offset) -} +mod windows { + use std::io; -#[cfg(windows)] -fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match fd.seek_read(buf, offset) { - Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), - Ok(n) => { - offset += n as u64; - buf = &mut buf[n..]; + pub fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match fd.seek_write(buf, offset) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + offset += n as u64; + buf = &buf[n..]; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), } - } - - Ok(()) -} -#[cfg(unix)] -#[inline] -fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { - fd.write_all_at(buf, offset) -} + Ok(()) + } -#[cfg(windows)] -fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match fd.seek_write(buf, offset) { - Ok(0) => return Err(io::ErrorKind::WriteZero.into()), - Ok(n) => { - offset += n as u64; - buf = &buf[n..]; + pub fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match fd.seek_read(buf, offset) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => { + offset += n as u64; + buf = &mut buf[n..]; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), } + + Ok(()) } - Ok(()) + pub async fn open_with_direct_io( + mut options: std::fs::OpenOptions, + path: impl AsRef, + ) -> io::Result { + use std::os::windows::fs::OpenOptionsExt as _; + + options + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) + .open(path) + } } From a375c7fa599a234e8fc8c03ec66cbd17d545cdd1 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 27 Aug 2026 10:46:36 +0200 Subject: [PATCH 13/19] WIP: simulator redesign --- Cargo.lock | 2 + crates/runtime-core/Cargo.toml | 7 +- crates/runtime-core/src/io/buf.rs | 178 +++++ crates/runtime-core/src/io/error.rs | 43 + crates/runtime-core/src/io/mod.rs | 95 +-- crates/runtime-core/src/lib.rs | 2 +- crates/runtime-core/src/sim/executor/io.rs | 101 --- crates/runtime-core/src/sim/executor/mod.rs | 37 +- crates/runtime-core/src/sim/io/executor.rs | 844 ++++++++++++++++++++ crates/runtime-core/src/sim/io/fs.rs | 8 +- crates/runtime-core/src/sim/io/mod.rs | 514 ++++++++---- crates/runtime-core/src/sim/io/op.rs | 563 ------------- crates/runtime/src/io/tokio.rs | 12 +- 13 files changed, 1470 insertions(+), 936 deletions(-) create mode 100644 crates/runtime-core/src/io/buf.rs create mode 100644 crates/runtime-core/src/io/error.rs delete mode 100644 crates/runtime-core/src/sim/executor/io.rs create mode 100644 crates/runtime-core/src/sim/io/executor.rs delete mode 100644 crates/runtime-core/src/sim/io/op.rs diff --git a/Cargo.lock b/Cargo.lock index c38b884a6ac..e910f3b4c42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8344,8 +8344,10 @@ version = "2.8.3" dependencies = [ "async-task", "futures-channel", + "slab", "spin", "thiserror 2.0.17", + "tokio", "zerocopy", ] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 6b644b6ab92..5b2b8571106 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -11,11 +11,16 @@ workspace = true [features] default = [] -sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] +alloc = [] +sim = ["alloc", "dep:async-task", "dep:futures-channel", "dep:slab", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } +slab = { version = "0.4", default-features = false, optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } thiserror = { version = "2.0", default-features = false } zerocopy = "0.8" + +[dev-dependencies] +tokio.workspace = true diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-core/src/io/buf.rs new file mode 100644 index 00000000000..16a8123b5c5 --- /dev/null +++ b/crates/runtime-core/src/io/buf.rs @@ -0,0 +1,178 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::io::SECTOR_SIZE; + +/// Types that can be safely converted to and from sector-aligned byte slices. +pub trait AlignedBytes: Sized { + /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the + /// right alignment. + /// + /// The type must also not rely on drop glue, i.e. `!core::mem::needs_drop()`. + /// + /// NOTE: Associated constants are evaluated lazily -- add a free + /// + /// `const _: () = ::ASSERT_VALID_LAYOUT;` + /// + /// for each `T` that is supposed to be used as an `AlignedBytes`. + const ASSERT_VALID_LAYOUT: () = { + assert!(align_of::() == SECTOR_SIZE); + assert!(size_of::().is_multiple_of(SECTOR_SIZE)); + assert!(!core::mem::needs_drop::()); + }; + + /// Reinterpret `self` as a byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes(&self) -> &[u8]; + + /// Reinterpret `self` as a mutable byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes_mut(&mut self) -> &mut [u8]; + + /// Reinterpret a byte slice as `Self`. + /// + /// The slice must be of length `size_of::()`. + /// + /// NOTE: Any slice of the right size, but consisting of only `0` (zero) + /// bytes can be converted to `Self`. It is the caller's responsibility to + /// validate the returned type as per the application's invariants. + /// + /// # Panics + /// + /// Panics if `b.len() != size_of::()`. + fn from_bytes(b: &[u8]) -> Self; +} + +impl AlignedBytes for T { + fn as_bytes(&self) -> &[u8] { + ::as_bytes(self) + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + ::as_mut_bytes(self) + } + + fn from_bytes(b: &[u8]) -> Self { + Self::read_from_bytes(b).unwrap() + } +} + +#[cfg(feature = "alloc")] +mod boxed { + use alloc::boxed::Box; + use core::{alloc::Layout, any::TypeId, ptr::NonNull}; + + use crate::io::AlignedBytes; + + /// A type-erased [AlignedBytes] heap allocation. + pub struct ErasedBox { + ptr: NonNull, + len: usize, + layout: Layout, + ty: TypeId, + } + + impl ErasedBox { + /// Create an [ErasedBox] from `B` by allocating a new [Box]. + pub fn from_aligned(b: B) -> Self { + Self::from_aligned_box(Box::new(b)) + } + + /// Create an [ErasedBox] from an already-boxed `B`. + pub fn from_aligned_box(b: Box) -> Self { + let () = B::ASSERT_VALID_LAYOUT; + + let ptr = Box::into_raw(b); + Self { + ptr: NonNull::new(ptr.cast()).unwrap(), + len: size_of::(), + layout: Layout::from_size_align(size_of::(), align_of::()).unwrap(), + ty: TypeId::of::(), + } + } + + /// Reify `B` via casting. + pub fn into_aligned(self) -> B { + *Self::into_aligned_box(self) + } + + /// Reify `B` via casting, without unboxing. + pub fn into_aligned_box(self) -> Box { + assert_eq!(self.len, size_of::()); + assert_eq!(self.ty, TypeId::of::()); + + let boxed = unsafe { Box::from_raw(self.ptr.as_ptr().cast::()) }; + // Prevent drop, which would deallocate. + core::mem::forget(self); + + boxed + } + + pub fn as_ptr(&self) -> ErasedBoxPtr { + ErasedBoxPtr { + ptr: self.ptr.as_ptr(), + len: self.len, + } + } + } + + impl Drop for ErasedBox { + fn drop(&mut self) { + unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } + } + } + + pub struct ErasedBoxPtr { + ptr: *mut u8, + len: usize, + } + + impl ErasedBoxPtr { + pub fn as_bytes(&mut self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr, self.len) } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) } + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[repr(C, align(4096))] + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct Trivial([u8; 4096]); + + impl AlignedBytes for Trivial { + fn as_bytes(&self) -> &[u8] { + &self.0 + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 + } + + fn from_bytes(b: &[u8]) -> Self { + assert_eq!(b.len(), size_of::()); + let mut a = [0; 4096]; + a.copy_from_slice(b); + Self(a) + } + } + + #[test] + fn roundtrip_preserves_value() { + let t = Trivial([32; 4096]); + + let erased = ErasedBox::from_aligned(t); + let reified = erased.into_aligned::(); + + assert_eq!(reified, t); + } + } +} +#[cfg(feature = "alloc")] +pub use boxed::{ErasedBox, ErasedBoxPtr}; diff --git a/crates/runtime-core/src/io/error.rs b/crates/runtime-core/src/io/error.rs new file mode 100644 index 00000000000..0a06c1ada98 --- /dev/null +++ b/crates/runtime-core/src/io/error.rs @@ -0,0 +1,43 @@ +/// An error `E`, along with auxiliary data `T`. +/// +/// `T` is usually a buffer of type [AlignedBytes], whose ownership is +/// transferred back to the caller when an error occurs. +/// +/// As this type signifies an error condition, the contents of `T` are +/// unspecified. +/// +/// [AlignedBytes]: crate::io::buf::AlignedBytes +#[derive(Debug)] +pub struct ErrorWith { + pub error: E, + pub with: T, +} + +impl ErrorWith { + /// Map a type-changing function over `self.error`. + pub fn map_err(self, f: impl FnOnce(E) -> F) -> ErrorWith { + ErrorWith { + error: f(self.error), + with: self.with, + } + } + + /// Map a type-changing function over `self.with`. + pub fn map_with(self, f: impl FnOnce(T) -> U) -> ErrorWith { + ErrorWith { + error: self.error, + with: f(self.with), + } + } + + /// Extract `self.error`, discarding `self.with`. + pub fn into_err(self) -> E { + self.error + } + + /// Convert from `&ErrorWith` to `ErrorWith<&E, &T>`. + pub fn as_ref(&self) -> ErrorWith<&E, &T> { + let Self { ref error, ref with } = *self; + ErrorWith { error, with } + } +} diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index 7b54b1dd39b..d5a946b14ca 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -1,74 +1,27 @@ -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +mod buf; +pub use buf::AlignedBytes; +#[cfg(feature = "alloc")] +pub use buf::{ErasedBox, ErasedBoxPtr}; + +mod error; +pub use error::ErrorWith; /// Size in bytes of a disk sector. pub const SECTOR_SIZE: usize = 4096; -/// Types that can be safely converted to and from sector-aligned byte slices. -pub trait AlignedBytes: Sized { - /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the - /// right aligment. - /// - /// NOTE: Associated constants are evaluated lazily -- add a free - /// - /// `const _: () = ::ASSERT_VALID_LAYOUT;` - /// - /// for each `T` that is supposed to be used as an `AlignedBytes`. - const ASSERT_VALID_LAYOUT: () = { - assert!(align_of::() == SECTOR_SIZE); - assert!(size_of::().is_multiple_of(SECTOR_SIZE)); - }; - - /// Reinterpret `self` as a byte slice. - /// - /// The returned slice will be of length `size_of::()`. - fn as_bytes(&self) -> &[u8]; - - /// Reinterpret `self` as a mutable byte slice. - /// - /// The returned slice will be of length `size_of::()`. - fn as_bytes_mut(&mut self) -> &mut [u8]; - - /// Reinterpret a byte slice as `Self`. - /// - /// The slice must be of length `size_of::()`. - /// - /// NOTE: Any slice of the right size, but consisting of only `0` (zero) - /// bytes can be converted to `Self`. It is the caller's responsibility to - /// validate the returned type as per the application's invariants. - /// - /// # Panics - /// - /// Panics if `b.len() != size_of::()`. - fn from_bytes(b: &[u8]) -> Self; +/// Subset of the `statx` metadata. +#[derive(Debug)] +#[non_exhaustive] +pub struct Statx { + pub size: u64, } -impl AlignedBytes for T { - fn as_bytes(&self) -> &[u8] { - ::as_bytes(self) - } - - fn as_bytes_mut(&mut self) -> &mut [u8] { - ::as_mut_bytes(self) - } - - fn from_bytes(b: &[u8]) -> Self { - Self::read_from_bytes(b).unwrap() +impl Statx { + pub fn from_size(size: u64) -> Self { + Self { size } } } -/// An error `E`, along with auxiliary data `T`. -/// -/// `T` is usually a buffer of type [AlignedBytes], whose ownership is -/// transferred back to the caller when an error occurs. -/// -/// As this type signifies an error condition, the contents of `T` are -/// unspecified. -#[derive(Debug)] -pub struct ErrorWith { - pub error: E, - pub with: T, -} - /// The canonical, low-level I/O API. /// /// Currently only supports file I/O, but eventually all I/O performed by @@ -98,15 +51,17 @@ pub trait SpacetimeIO { /// from being `no_std`. /// /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 - type Error; + type Error: core::error::Error; + /// The completion [Future] of all methods in this trait. + type Completion: Future + Unpin; /// Open the file at `path`. fn open_file(&self, path: &str) -> Self::Completion>; - /// Create the file at `path` and allocate `len` bytes. + /// Create the file at `path`. /// /// Returns an error if the file already exists. - fn create_file(&self, path: &str, len: u64) -> Self::Completion>; + fn create_file(&self, path: &str) -> Self::Completion>; /// Write `buf` to `fd` at `offset`. /// @@ -143,11 +98,15 @@ pub trait SpacetimeIO { /// Call `fdatasync(2)` on `fd`. fn fdatasync(&self, fd: Self::Fd) -> Self::Completion>; - /// Allocate `additional` bytes for the file `fd`. - fn reserve(&self, fd: Self::Fd, additional: u64) -> Self::Completion>; + /// Allocate `total` bytes for the file `fd`. + /// + /// Implementations must ensure that attempts to shrink the file result in + /// an error. The operation should succeed if the file's size is already + /// `total`. + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion>; /// Determine the length of the file `fd`. /// /// This should not depend on `fsync`, i.e. `statx`. See `std::io::Seek::stream_len`. - fn length(&self, fd: Self::Fd) -> Self::Completion>; + fn statx(&self, fd: Self::Fd) -> Self::Completion>; } diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index e35d042ea9a..8a841c22036 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -#[cfg(feature = "sim")] +#[cfg(any(feature = "sim", feature = "alloc"))] extern crate alloc; #[cfg(test)] extern crate std; diff --git a/crates/runtime-core/src/sim/executor/io.rs b/crates/runtime-core/src/sim/executor/io.rs deleted file mode 100644 index ca1831978bf..00000000000 --- a/crates/runtime-core/src/sim/executor/io.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::sim::{io::SimulatorIO, Rng}; - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Config { - /// The max number of submissions to run per [Driver::tick]. - pub max_submissions_per_tick: usize, - /// The max number of completions to finish per [Driver::tick]. - pub max_completions_per_tick: usize, - /// Submission reordering probability. - /// - /// Describes the probability by which to select the next submission queue - /// entry randomly, as opposed to the oldest entry in the queue. - pub prob_reorder_submissions: f64, - /// Completion reordering probability. - /// - /// Describes the probability by which to select the next completion queue - /// entry randomly, as opposed to the oldest entry in the queue. - pub prob_reorder_completions: f64, - /// Probability by which to skip one submission queue entry. - /// - /// If skipped, the entry still counts towards `max_submissions_per_tick`. - pub prob_skip: f64, - /// Probability by which to cancel a submission queue entry. - /// - /// [crate::sim::io::op::Submission::cancel()] is called on the entry, which - /// may generate a completion. - pub prob_cancel: f64, -} - -impl Default for Config { - fn default() -> Self { - Self { - max_submissions_per_tick: 1, - max_completions_per_tick: 1, - prob_reorder_submissions: 0.0, - prob_reorder_completions: 0.0, - prob_skip: 0.0, - prob_cancel: 0.0, - } - } -} - -pub struct Driver { - io: SimulatorIO, - config: Config, -} - -impl Driver { - pub fn new(config: Config) -> Self { - Self { - io: <_>::default(), - config, - } - } - - /// Advance the I/O simulator according the [Config]. - /// - /// Returns `true` if progress has been made, or there are pending entries - /// in either the submission or completion queue. - pub fn tick(&self, rng: &Rng) -> bool { - let mut progress = false; - for _ in 0..self.config.max_submissions_per_tick { - if !rng.buggify_with_prob(self.config.prob_skip) { - let sqe = if rng.buggify_with_prob(self.config.prob_reorder_submissions) { - self.io.random_submission(rng) - } else { - self.io.next_submission() - }; - - if let Some(sqe) = sqe { - if rng.buggify_with_prob(self.config.prob_cancel) { - sqe.cancel(); - } else { - self.io.execute(sqe); - } - progress = true; - } - } - } - - for _ in 0..self.config.max_completions_per_tick { - let cqe = if rng.buggify_with_prob(self.config.prob_reorder_completions) { - self.io.random_completion(rng) - } else { - self.io.next_completion() - }; - - if let Some(cqe) = cqe { - cqe.complete(); - progress = true - } - } - - progress |= self.io.pending(); - progress - } - - pub fn io(&self) -> &SimulatorIO { - &self.io - } -} diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index 1913329b2e2..a0fbca1bf7c 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -10,12 +10,8 @@ use core::{ use spin::Mutex; -use crate::sim::io::SimulatorIO; - use super::{time::TimeHandle, Rng}; -mod io; - mod task; use task::Abortable; pub use task::{AbortHandle, JoinError, JoinHandle}; @@ -25,23 +21,11 @@ type Runnable = async_task::Runnable; #[derive(Clone, Copy, Debug, PartialEq)] pub struct RuntimeConfig { pub seed: u64, - pub io: Option, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed, io: None } - } - - pub fn enable_io(self) -> Self { - Self { - io: Some(self.io.unwrap_or_default()), - ..self - } - } - - pub fn with_io_config(self, io: Option) -> Self { - Self { io, ..self } + Self { seed } } } @@ -161,12 +145,6 @@ impl Runtime { } } - // TODO: This is a stopgap to allow submission of I/O tasks. We probably - // want the user-facing API to hide this. - pub fn io(&self) -> Option<&SimulatorIO> { - self.executor.io.as_ref().map(|driver| driver.io()) - } - /// Drive a top-level future to completion on the simulation executor. /// /// While the future runs, spawned tasks share the same deterministic @@ -382,7 +360,6 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, - io: Option, } impl Executor { @@ -398,7 +375,6 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), - io: config.io.map(io::Driver::new), } } @@ -515,7 +491,6 @@ impl Executor { loop { self.run_all_ready(); - let pending_io = self.drive_io(); if task.is_finished() { let waker = Waker::noop(); return match Pin::new(&mut task).poll(&mut Context::from_waker(waker)) { @@ -524,7 +499,7 @@ impl Executor { }; } - if self.time.wake_next_timer() || pending_io { + if self.time.wake_next_timer() { continue; } @@ -552,14 +527,6 @@ impl Executor { } } - fn drive_io(&self) -> bool { - if let Some(io) = &self.io { - io.tick(&self.rng) - } else { - false - } - } - /// Look up the record for a node, panicking if the node is unknown. fn node_record(&self, node: NodeId) -> Arc { self.nodes diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs new file mode 100644 index 00000000000..9135fa45c7c --- /dev/null +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -0,0 +1,844 @@ +#![allow(unused)] + +use alloc::{ + boxed::Box, + collections::{btree_map, BTreeMap, VecDeque}, + vec::Vec, +}; +use core::result::Result; + +use crate::{ + io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, + sim::{ + io::{fs, Error, Instant}, + Rng, + }, +}; + +#[derive(Clone, Copy)] +pub enum LinkKind { + Soft, + Hard, +} + +pub struct Sqe { + inner: SqeInner, + link: Option, + user_data: Option, +} + +impl Sqe { + pub fn link(mut self, kind: Option) -> Self { + self.link = kind; + self + } + + pub fn is_linked(&self) -> bool { + self.link.is_some() + } + + pub fn attach(mut self, user_data: T) -> Self { + self.user_data.replace(user_data); + self + } + + pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Write { fd, buf, offset }.into() + } + + pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Read { fd, buf, offset }.into() + } + + pub fn open(path: impl AsRef) -> Self { + Open { + path: path.as_ref().into(), + } + .into() + } + + pub fn create(path: impl AsRef) -> Self { + Create { + path: path.as_ref().into(), + } + .into() + } + + pub fn stat(fd: fs::File) -> Self { + Stat { fd }.into() + } + + pub fn fallocate(fd: fs::File, len: u64) -> Self { + Fallocate { fd, total_len: len }.into() + } + + pub fn fsync(fd: fs::File) -> Self { + Fsync { fd }.into() + } + + pub fn fdatasync(fd: fs::File) -> Self { + Fdatasync { fd }.into() + } + + pub fn noop() -> Self { + SqeInner::Noop.into() + } +} + +impl> From for Sqe { + fn from(inner: U) -> Self { + Self { + inner: inner.into(), + link: None, + user_data: None, + } + } +} + +enum SqeInner { + Write(Write), + Read(Read), + Open(Open), + Create(Create), + Stat(Stat), + Fallocate(Fallocate), + Fsync(Fsync), + Fdatasync(Fdatasync), + Noop, +} + +impl SqeInner { + fn cancel(self, user_data: Option) -> Cqe { + match self { + SqeInner::Write(..) => Cqe::Write { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Read(..) => Cqe::Read { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Open(..) => Cqe::Open { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Create(..) => Cqe::Create { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Stat(..) => Cqe::Stat { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fallocate(..) => Cqe::Fallocate { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fsync(..) => Cqe::Fsync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fdatasync(..) => Cqe::Fdatasync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Noop => Cqe::Noop { + result: Err(Error::Cancelled), + user_data, + }, + } + } + + fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { + match self { + SqeInner::Write(mut sqe) => { + let Write { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Operation::WriteSector { + sqe: sqe_id, + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }) + .collect::>(); + let op_count = ops.len(); + let write = InFlightInner::Write { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (write, ops) + } + SqeInner::Read(mut sqe) => { + let Read { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Operation::ReadSector { + sqe: sqe_id, + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }) + .collect::>(); + let op_count = ops.len(); + let read = InFlightInner::Read { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (read, ops) + } + SqeInner::Open(sqe) => ( + InFlightInner::Open { sqe }, + alloc::vec![Operation::Open { sqe: sqe_id }], + ), + SqeInner::Create(sqe) => ( + InFlightInner::Create { sqe }, + alloc::vec![Operation::Create { sqe: sqe_id }], + ), + SqeInner::Stat(sqe) => ( + InFlightInner::Stat { sqe }, + alloc::vec![Operation::Stat { sqe: sqe_id }], + ), + SqeInner::Fallocate(sqe) => ( + InFlightInner::Fallocate { sqe }, + alloc::vec![Operation::Fallocate { sqe: sqe_id }], + ), + SqeInner::Fsync(sqe) => ( + InFlightInner::Fsync { sqe }, + alloc::vec![Operation::Fsync { sqe: sqe_id }], + ), + SqeInner::Fdatasync(sqe) => ( + InFlightInner::Fdatasync { sqe }, + alloc::vec![Operation::Fdatasync { sqe: sqe_id }], + ), + SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), + } + } +} + +impl From for SqeInner { + fn from(inner: Write) -> Self { + Self::Write(inner) + } +} + +impl From for SqeInner { + fn from(inner: Read) -> Self { + Self::Read(inner) + } +} + +impl From for SqeInner { + fn from(inner: Open) -> Self { + Self::Open(inner) + } +} + +impl From for SqeInner { + fn from(inner: Create) -> Self { + Self::Create(inner) + } +} + +impl From for SqeInner { + fn from(inner: Stat) -> Self { + Self::Stat(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fallocate) -> Self { + Self::Fallocate(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fsync) -> Self { + Self::Fsync(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fdatasync) -> Self { + Self::Fdatasync(inner) + } +} + +struct Write { + fd: fs::File, + buf: ErasedBoxPtr, + offset: u64, +} + +struct Read { + fd: fs::File, + buf: ErasedBoxPtr, + offset: u64, +} + +struct Open { + path: Box, +} + +struct Create { + path: Box, +} + +struct Stat { + fd: fs::File, +} + +struct Fallocate { + fd: fs::File, + total_len: u64, +} + +struct Fsync { + #[allow(unused)] + fd: fs::File, +} + +struct Fdatasync { + #[allow(unused)] + fd: fs::File, +} + +#[derive(Debug)] +pub enum Cqe { + Write { + result: Result, + user_data: Option, + }, + Read { + result: Result, + user_data: Option, + }, + Open { + result: Result, + user_data: Option, + }, + Create { + result: Result, + user_data: Option, + }, + Stat { + result: Result, + user_data: Option, + }, + Fallocate { + result: Result<(), Error>, + user_data: Option, + }, + Fsync { + result: Result<(), Error>, + user_data: Option, + }, + Fdatasync { + result: Result<(), Error>, + user_data: Option, + }, + Noop { + result: Result<(), Error>, + user_data: Option, + }, +} + +impl Cqe { + pub fn user_data(&self) -> &Option { + match self { + Self::Write { user_data, .. } + | Self::Read { user_data, .. } + | Self::Open { user_data, .. } + | Self::Create { user_data, .. } + | Self::Stat { user_data, .. } + | Self::Fallocate { user_data, .. } + | Self::Fsync { user_data, .. } + | Self::Fdatasync { user_data, .. } + | Self::Noop { user_data, .. } => user_data, + } + } +} + +type SqeId = usize; + +enum Operation { + WriteSector { + sqe: SqeId, + page_offset: usize, + buf_offset: usize, + }, + ReadSector { + sqe: SqeId, + page_offset: usize, + buf_offset: usize, + }, + Open { + sqe: SqeId, + }, + Create { + sqe: SqeId, + }, + Stat { + sqe: SqeId, + }, + Fallocate { + sqe: SqeId, + }, + Fsync { + sqe: SqeId, + }, + Fdatasync { + sqe: SqeId, + }, + Noop { + sqe: SqeId, + }, +} + +struct Blocked { + link: LinkKind, + sqe: SqeInner, + user_data: Option, +} + +struct InFlight { + inner: InFlightInner, + blocked: VecDeque>, + user_data: Option, +} + +enum InFlightInner { + Write { + sqe: Write, + op_count: usize, + results: Vec>, + }, + Read { + sqe: Read, + op_count: usize, + results: Vec>, + }, + Open { + sqe: Open, + }, + Create { + sqe: Create, + }, + Stat { + sqe: Stat, + }, + Fallocate { + sqe: Fallocate, + }, + Fsync { + sqe: Fsync, + }, + Fdatasync { + sqe: Fdatasync, + }, + Noop, +} + +pub enum WriteFault { + /// Misdirect the write to an arbitrary page offset in the file. + Misdirected { page_offset: usize }, + /// Report the write as successful, but don't write anything. + Lost, + /// Report the write as successful, but write less bytes than requested. + Short { write_bytes: usize }, + /// Delay the write until at least `deadline`. + Delayed { deadline: Instant }, + /// Execute the side effects, but never report completion. + NoCompletion, + /// Report an error without executing side effects. + Error(Error), +} + +pub trait FaultInjector { + fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; +} + +pub struct Executor { + submissions: VecDeque>, + completions: VecDeque>, + + in_flight: [Option>; MAX_INFLIGHT], + executing: VecDeque, + + fstree: BTreeMap, fs::File>, +} + +impl Executor { + pub fn with_capacity(capacity: usize) -> Self { + Self { + submissions: VecDeque::with_capacity(capacity), + completions: VecDeque::with_capacity(capacity), + in_flight: core::array::from_fn(|_| None), + executing: VecDeque::new(), + fstree: BTreeMap::new(), + } + } + + pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> + where + Batch: IntoIterator>, + Batch::IntoIter: ExactSizeIterator, + { + let sqes = sqes.into_iter(); + if self.submissions.len() + sqes.len() >= self.submissions.capacity() { + Err(sqes) + } else { + self.submissions.extend(sqes); + Ok(()) + } + } + + fn complete(&mut self, cqe: Cqe) { + assert!( + self.completions.len() < self.completions.capacity(), + "completion queue overflow" + ); + self.completions.push_back(cqe); + } + + pub fn completed(&mut self) -> impl Iterator> { + self.completions.drain(..) + } + + pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { + let mut progress = self.schedule(); + progress |= self.execute(rng, now); + progress + } + + fn schedule(&mut self) -> bool { + let mut progress = false; + + // Fill free execution slots. + for (id, slot) in self.in_flight.iter_mut().filter(|f| f.is_none()).enumerate() { + let Some(sqe) = self.submissions.pop_front() else { + break; + }; + + // If the sqe is linked, pop the whole chain. + // Links of sqes not submitted in the same batch are ignored. + let mut successors = VecDeque::new(); + if let Some(link) = sqe.link { + let mut link_kind = link; + while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { + successors.push_back(Blocked { + link: link_kind, + sqe: inner, + user_data, + }); + match link { + Some(kind) => link_kind = kind, + None => break, + } + } + } + + let (in_flight, ops) = sqe.inner.schedule(id); + self.executing.extend(ops); + slot.replace(InFlight { + inner: in_flight, + blocked: successors, + user_data: sqe.user_data, + }); + + progress = true + } + + progress + } + + fn execute(&mut self, rng: &Rng, now: Instant) -> bool { + if self.executing.is_empty() { + return false; + } + if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { + match op { + Operation::WriteSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { + let InFlight { + inner: + InFlightInner::Write { + sqe: Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &buf.as_bytes()[buf_offset..end]; + let result = fd.write_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Write { + sqe: Write { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Write { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + Operation::ReadSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { + let InFlight { + inner: + InFlightInner::Read { + sqe: Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + let result = fd.read_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Read { + sqe: Read { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invlid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + Operation::Open { sqe } => { + let InFlight { + inner: InFlightInner::Open { sqe: Open { path } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected open") + }; + + let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Create { sqe } => { + let InFlight { + inner: InFlightInner::Create { sqe: Create { path } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected create") + }; + + let result = match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Stat { sqe } => { + let InFlight { + inner: InFlightInner::Stat { sqe: Stat { fd } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected stat") + }; + + self.complete(Cqe::Stat { + result: Ok(Statx { size: fd.len() }), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fallocate { sqe } => { + let InFlight { + inner: + InFlightInner::Fallocate { + sqe: Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fallocate") + }; + + self.complete(Cqe::Fallocate { + result: fd.set_len(total_len).map_err(Error::from), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fsync { sqe } => { + // TODO: Do something fallible with fd. + let InFlight { + inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fsync") + }; + + self.complete(Cqe::Fsync { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fdatasync { sqe } => { + // TODO: Do something fallible with fd. + let InFlight { + inner: + InFlightInner::Fdatasync { + sqe: Fdatasync { fd: _ }, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fdatasync") + }; + + self.complete(Cqe::Fdatasync { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Noop { sqe } => { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected noop") + }; + + self.complete(Cqe::Noop { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + } + + return true; + } + + false + } + + fn schedule_linked( + &mut self, + in_flight_slot: SqeId, + prev_succeeded: bool, + mut blocked: VecDeque>, + ) { + if let Some(Blocked { + link, + sqe: next, + user_data, + }) = blocked.pop_front() + { + match (link, prev_succeeded) { + (LinkKind::Soft, false) => { + self.complete(next.cancel(user_data)); + for Blocked { + link: _, + sqe: next, + user_data, + } in blocked + { + self.complete(next.cancel(user_data)); + } + } + (LinkKind::Soft, true) | (LinkKind::Hard, _) => { + let (inner, ops) = next.schedule(in_flight_slot); + self.executing.extend(ops); + self.in_flight[in_flight_slot].replace(InFlight { + inner, + blocked, + user_data, + }); + } + } + } + } +} diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index efa3d9ab755..93e1288fac3 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,6 +1,6 @@ use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cmp, + cmp, fmt, sync::atomic::{AtomicU64, Ordering}, }; @@ -54,6 +54,12 @@ pub struct File { len: Arc, } +impl fmt::Debug for File { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("File").field("len", &self.len).finish() + } +} + impl File { pub(super) fn new() -> Self { Self { diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index a86d4b0834a..d981ba38193 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -1,26 +1,33 @@ -use alloc::{ - boxed::Box, - collections::{BTreeMap, VecDeque}, - sync::Arc, -}; +use alloc::{boxed::Box, sync::Arc}; use core::{ - num::NonZeroUsize, pin::Pin, result::Result, task::{Context, Poll}, + time::Duration, }; use futures_channel::oneshot; +use slab::Slab; use crate::{ - io::{AlignedBytes, ErrorWith, SpacetimeIO}, + io::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}, sim::Rng, }; +mod executor; +use executor::{Cqe, Executor, Sqe}; + mod fs; -pub mod op; +pub use fs::File; pub use crate::io::SECTOR_SIZE; -pub use fs::File; + +/// Simulated clock measurement. +/// +/// In simulated time, an instant is actually a [Duration] since the time +/// instance was instantiated. To avoid confusion, we use the name "instant" to +/// convey that its semantics are that of the standard library type of the same +/// name. +pub type Instant = Duration; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -37,6 +44,8 @@ pub enum Error { /// Injected by the I/O driver. #[error("operation cancelled")] Cancelled, + #[error("submission queue overflow")] + SubmissionQueueOverflow, } impl From for Error { @@ -47,103 +56,239 @@ impl From for Error { #[derive(Clone, Default)] pub struct SimulatorIO { - // TODO: We make `SimulatorIO` `Send + Sync` for now, because - // [crate::sim::executor::Handle] is just `Arc`. This means that a - // future carrying a handle can't be `spawn`ed, because spawning requires - // the future to be `Send`. - // - // We should fix this at some point, so below can become `Rc>`. - inner: Arc>, + inner: Arc, } impl SimulatorIO { - /// Returns `true` if there are entries in either the submission or - /// completion queues. - pub fn pending(&self) -> bool { - let inner = self.inner.lock(); - inner.submissions.len() + inner.completions.len() > 0 - } - - /// Number of entries in the submission queue. - pub fn pending_submissions(&self) -> usize { - self.inner.lock().submissions.len() - } + pub fn tick(&self, rng: &Rng, now: Instant) -> bool { + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let mut buffers = self.inner.buffers.lock(); + + let mut progress = executor.tick(rng, now); + for cqe in executor.completed() { + let completion = pending.remove(cqe.user_data().unwrap()); + match cqe { + Cqe::Write { result, .. } => { + let CompletionHandle::Write { tx, buf_key } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let erased_buf = buffers.remove(buf_key); + let result = match result { + Ok(_written) => Ok(erased_buf), + Err(error) => Err(ErrorWith { + error, + with: erased_buf, + }), + }; + let _ = tx.send(result); + } + Cqe::Read { result, .. } => { + let CompletionHandle::Read { tx, buf_key } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let erased_buf = buffers.remove(buf_key); + let result = match result { + Ok(_written) => Ok(erased_buf), + Err(error) => Err(ErrorWith { + error, + with: erased_buf, + }), + }; + let _ = tx.send(result); + } + Cqe::Open { result, .. } => { + let CompletionHandle::Open { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Create { result, .. } => { + let CompletionHandle::Create { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Stat { result, .. } => { + let CompletionHandle::Stat { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fallocate { result, .. } => { + let CompletionHandle::Fallocate { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fsync { result, .. } => { + let CompletionHandle::Fsync { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fdatasync { result, .. } => { + let CompletionHandle::Fdatasync { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Noop { result, .. } => { + let CompletionHandle::Noop { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + } - /// Number of entries in the completion queue. - pub fn pending_completions(&self) -> usize { - self.inner.lock().completions.len() - } + progress |= true; + } - /// Run the submission at the front of the queue (if any), and complete the - /// completion at the front of the queue (if any). - pub fn tick(&self) -> bool { - self.inner.lock().tick() + progress } +} - /// Execute `sqe`. - pub fn execute(&self, sqe: Box) { - self.inner.lock().execute(sqe); - } +struct SimulatorInner { + executor: spin::Mutex>, + pending: spin::Mutex>, + buffers: Arc>>, +} - /// Remove and return the submission at the front of the queue, if any. - pub fn next_submission(&self) -> Option> { - self.inner.lock().next_submission() +impl Default for SimulatorInner { + fn default() -> Self { + Self { + executor: spin::Mutex::new(Executor::with_capacity(128)), + pending: <_>::default(), + buffers: <_>::default(), + } } +} - /// Remove and return a random submission, or `None` if the queue is empty. - pub fn random_submission(&self, rng: &Rng) -> Option> { - self.inner.lock().random_submission(rng) - } +#[must_use = "completions must be polled to completion"] +pub struct Completion(CompletionInner); - /// Remove and return the completion at the front of the queue, if any. - pub fn next_completion(&self) -> Option> { - self.inner.lock().next_completion() +impl Completion { + pub fn mapped( + rx: oneshot::Receiver>>, + map: fn(Result>) -> T, + ) -> Self { + Self(CompletionInner::Mapped { rx, map }) } +} - /// Remove and return a random completion, or `None` if the queue is empty. - pub fn random_completion(&self, rng: &Rng) -> Option> { - self.inner.lock().random_completion(rng) +impl From> for Completion { + fn from(rx: oneshot::Receiver) -> Self { + Self(CompletionInner::Direct { rx }) } +} - fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> Completion { - let (tx, rx) = oneshot::channel(); - self.inner.lock().submit(op(tx)); - Completion(rx) - } +impl Future for Completion { + type Output = T; - fn submit_all>>( - &self, - ops: impl FnOnce(oneshot::Sender) -> I, - ) -> Completion { - let (tx, rx) = oneshot::channel(); - let mut inner = self.inner.lock(); - ops(tx).for_each(|op| inner.submit(op)); - Completion(rx) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + Pin::new(&mut this.0).poll(cx) } } -#[must_use = "completions must be polled to completion"] -pub struct Completion(oneshot::Receiver); +enum CompletionInner { + Direct { + rx: oneshot::Receiver, + }, + Mapped { + rx: oneshot::Receiver>>, + map: fn(Result>) -> T, + }, +} -impl Future for Completion { +impl Future for CompletionInner { type Output = T; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::new(&mut self.as_mut().0).poll(cx).map(Result::unwrap) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match this { + Self::Direct { rx } => Pin::new(rx) + .poll(cx) + .map(|result| result.expect("lost completion sender")), + Self::Mapped { rx, map } => Pin::new(rx).poll(cx).map(|result| { + let result = result.expect("lost completion sender"); + map(result) + }), + } } } +enum CompletionHandle { + Write { + tx: oneshot::Sender>>, + buf_key: usize, + }, + Read { + tx: oneshot::Sender>>, + buf_key: usize, + }, + Open { + tx: oneshot::Sender>, + }, + Create { + tx: oneshot::Sender>, + }, + Stat { + tx: oneshot::Sender>, + }, + Fallocate { + tx: oneshot::Sender>, + }, + Fsync { + tx: oneshot::Sender>, + }, + Fdatasync { + tx: oneshot::Sender>, + }, + // TODO: We may use this for timeouts. + #[allow(unused)] + Noop { + tx: oneshot::Sender>, + }, +} + impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; type Completion = Completion; fn open_file(&self, path: &str) -> Self::Completion> { - self.submit(op::open_file(path)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::open(path).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Open { tx }); + } + } + + rx.into() } - fn create_file(&self, path: &str, len: u64) -> Self::Completion> { - self.submit(op::create_file(path, len)) + fn create_file(&self, path: &str) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::create(path).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Create { tx }); + } + } + + rx.into() } fn write_all_at( @@ -152,16 +297,29 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let () = B::ASSERT_VALID_LAYOUT; - - if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit(op::ready(Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }))) - } else { - self.submit_all(op::write_at(fd, buf, offset)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + let erased_buf = ErasedBox::from_aligned(buf); + let buf_ptr = erased_buf.as_ptr(); + + match executor.submit([Sqe::write(fd, buf_ptr, offset).attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: erased_buf, + })) + .unwrap_or_else(|_| unreachable!("rx is still alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(erased_buf); + pending_entry.insert(CompletionHandle::Write { tx, buf_key }); + } } + + Completion::mapped(rx, reify) } fn read_exact_at( @@ -170,106 +328,151 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let () = B::ASSERT_VALID_LAYOUT; - - if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit(op::ready(Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }))) - } else { - self.submit_all(op::read_at(fd, buf, offset)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + let erased_buf = ErasedBox::from_aligned(buf); + let buf_ptr = erased_buf.as_ptr(); + + match executor.submit([Sqe::read(fd, buf_ptr, offset).attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: erased_buf, + })) + .unwrap_or_else(|_| unreachable!("rx is still alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(erased_buf); + pending_entry.insert(CompletionHandle::Read { tx, buf_key }); + } } - } - fn fsync(&self, _fd: Self::Fd) -> Self::Completion> { - self.submit(op::ready(Ok(()))) + Completion::mapped(rx, reify) } - fn fdatasync(&self, _fd: Self::Fd) -> Self::Completion> { - self.submit(op::ready(Ok(()))) - } + fn fsync(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); - fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { - assert!(total >= fd.len()); - self.submit(op::set_len(fd, total)) - } + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::fsync(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fsync { tx }); + } + } - fn length(&self, fd: Self::Fd) -> Self::Completion> { - self.submit(op::get_len(fd)) + rx.into() } -} -#[derive(Default)] -struct SimulatorIOInner { - files: BTreeMap, fs::File>, - submissions: VecDeque>, - completions: VecDeque>, -} + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); -impl SimulatorIOInner { - fn tick(&mut self) -> bool { - let mut progress = false; - if let Some(sqe) = self.submissions.pop_front() { - if let Some(cqe) = sqe.execute(&mut self.files) { - self.completions.push_back(cqe); + match executor.submit([Sqe::fdatasync(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fdatasync { tx }); } - progress = true; - } - if let Some(cqe) = self.completions.pop_front() { - cqe.complete(); - progress = true; } - progress + rx.into() } - fn execute(&mut self, sqe: Box) { - if let Some(cqe) = sqe.execute(&mut self.files) { - self.completions.push_back(cqe); + fn reserve(&self, fd: Self::Fd, total_size: u64) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::fallocate(fd, total_size).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fallocate { tx }); + } } - } - fn next_submission(&mut self) -> Option> { - self.submissions.pop_front() + rx.into() } - fn random_submission(&mut self, rng: &Rng) -> Option> { - let len = NonZeroUsize::new(self.submissions.len())?; - self.submissions.remove(rng.index(len.get())) - } + fn statx(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); - fn next_completion(&mut self) -> Option> { - self.completions.pop_front() - } + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); - fn random_completion(&mut self, rng: &Rng) -> Option> { - let len = NonZeroUsize::new(self.completions.len())?; - self.completions.remove(rng.index(len.get())) + match executor.submit([Sqe::stat(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Stat { tx }); + } + } + + rx.into() } +} - fn submit(&mut self, op: Box) { - self.submissions.push_back(op); +fn reify( + result: Result>, +) -> Result> { + match result { + Ok(erased) => Ok(erased.into_aligned::()), + Err(ErrorWith { error, with }) => Err(ErrorWith { + error, + with: with.into_aligned::(), + }), } } #[cfg(test)] mod tests { - use crate::sim::{Runtime, RuntimeConfig}; + use crate::sim::{time::TimeHandle, GlobalRng}; use super::*; + struct Runtime { + rt: tokio::runtime::LocalRuntime, + io: SimulatorIO, + rng: Rng, + time: TimeHandle, + } + + impl Runtime { + fn new() -> Self { + Self { + rt: tokio::runtime::Builder::new_current_thread() + .build_local(<_>::default()) + .unwrap(), + io: SimulatorIO::default(), + rng: GlobalRng::new(0), + time: TimeHandle::default(), + } + } + + fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { + let fut = self.rt.spawn_local(f(&self.io)); + while self.io.tick(&self.rng, self.time.now()) {} + self.rt.block_on(fut).unwrap() + } + } + #[test] fn create_file() { - let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); - let io = rt.io().cloned().unwrap(); - - let fd = rt - .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) - .unwrap(); - assert_eq!(fd.len(), 2 * SECTOR_SIZE as u64); + let rt = Runtime::new(); + rt.run(|io| io.create_file("/data/test")).unwrap(); } + #[derive(Debug)] #[repr(C, align(4096))] struct Buf([u8; 2 * SECTOR_SIZE]); @@ -298,21 +501,14 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); - let io = rt.io().cloned().unwrap(); + let rt = Runtime::new(); - let fd = rt - .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) - .unwrap(); + let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); let mut buf = rt - .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) - .map_err(|ErrorWith { error, .. }| error) + .run(|io| io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) .unwrap(); buf.clear(); - let buf = rt - .block_on(io.read_exact_at(fd, buf, 0)) - .map_err(|ErrorWith { error, .. }| error) - .unwrap(); + let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); assert!(buf.0.iter().all(|&b| b == 22)); } diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs deleted file mode 100644 index 0e855313819..00000000000 --- a/crates/runtime-core/src/sim/io/op.rs +++ /dev/null @@ -1,563 +0,0 @@ -use core::{any::Any, iter::Scan, ops::Range}; - -use alloc::{ - boxed::Box, - collections::{btree_map, BTreeMap}, - sync::Arc, -}; -use futures_channel::oneshot; - -use super::{fs, Error}; -use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; - -/// An operation that can be submitted to the [super::SimulatorIO] driver. -pub trait Submission: Send + Any { - /// Run the operations with mutable access to the currently registered - /// [fs::File]s. - /// - /// If the operation is done, a [Completion] is returned in a `Some`. - /// `None` may be returned if: - /// - /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. - /// - The submission is a [Noop]. - /// - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; - - /// Cancel the operation instead of executing it. - /// - /// This will generate a [Completion] with the result [Error::Cancelled], - /// unless: - /// - /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. - /// - The submission is a [Noop]. - /// - fn cancel(self: Box) -> Option>; -} - -/// An object containing the result of executing a [Submission], as well as a -/// handle to resolve a future waiting on the outcome of the operation. -pub trait Completion: Send { - fn success(&self) -> bool; - /// Resolve the future waiting on the outcome of the operation. - fn complete(self: Box); -} - -/// A channel to resolve a future waiting on the outcome of a submitted -/// operation. -pub type OnComplete = oneshot::Sender; - -pub type ScanState = (fs::File, usize, Arc>>); -pub type PageWrites = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; -pub type PageReads = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; - -pub type WriteAtResult = Result>; - -/// Write the contents of `buf` to `fd` at `offset`. -/// -/// This operation is split into multiple writes to individual pages. The -/// `on_complete` future resolves only after all page writes completed. -/// -/// Ownership of `buf` is transferred back when the operation completes. -pub fn write_at( - fd: fs::File, - buf: B, - offset: u64, -) -> impl FnOnce(OnComplete>) -> PageWrites { - move |on_complete| { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { - let op = WritePage { - fd: fd.clone(), - file_page: *first_page + buf_page, - buf_page, - state: state.clone(), - }; - - Some(Box::new(op)) - }) - } -} - -pub type ReadAtResult = Result>; - -/// Fill `buf` by reading from `fd` at `offset`. -/// -/// This operation is split into multple reads from the individual pages needed -/// to fill `buf`. The `on_complete` future resolves only after all page reads -/// completed. -/// -/// Ownership of `buf` is transferred back when the operation completes. -pub fn read_at( - fd: fs::File, - buf: B, - offset: u64, -) -> impl FnOnce(OnComplete>) -> PageReads { - move |on_complete| { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { - let op = ReadPage { - fd: fd.clone(), - file_page: *first_page + buf_page, - buf_page, - state: state.clone(), - }; - - Some(Box::new(op)) - }) - } -} - -/// Open file at `path`. -pub fn open_file(path: &str) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| { - Box::new(OpenFile { - path: path.into(), - on_complete, - }) - } -} - -/// Create a new file at `path` and allocate `len` space for it. -pub fn create_file(path: &str, len: u64) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| { - Box::new(CreateFile { - path: path.into(), - len, - on_complete, - }) - } -} - -/// Get the length of the file `fd`. -pub fn get_len(fd: fs::File) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| Box::new(GetLen { fd, on_complete }) -} - -/// Set the length of the file `fd`. -pub fn set_len(fd: fs::File, len: u64) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| Box::new(SetLen { fd, len, on_complete }) -} - -struct GenericCompletion { - success: bool, - result: T, - on_complete: OnComplete, -} - -fn completion(success: bool, result: T, on_complete: OnComplete) -> Box { - Box::new(GenericCompletion { - success, - result, - on_complete, - }) -} - -impl Completion for GenericCompletion { - fn success(&self) -> bool { - self.success - } - - fn complete(self: Box) { - let Self { - result, on_complete, .. - } = *self; - let _ = on_complete.send(result); - } -} - -/// [Submission] created by [noop]. -pub(crate) struct Noop; - -impl Submission for Noop { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - None - } - - fn cancel(self: Box) -> Option> { - None - } -} - -/// An operation that does nothing. -/// -/// Note that no completion is associated with a noop, but the submission still -/// occupies a slot in the submission queue. -pub fn noop() -> Box { - Box::new(Noop) -} - -/// [Submission] created by [ready]. -pub(crate) struct Ready(Box); - -impl Submission for Ready { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self(completion) = *self; - Some(completion) - } - - fn cancel(self: Box) -> Option> { - let Self(completion) = *self; - Some(completion) - } -} - -/// An operation that is already complete with `result`. -pub fn ready(result: T) -> impl FnOnce(OnComplete) -> Box { - move |on_complete| Box::new(Ready(completion(true, result, on_complete))) -} - -/// [Submission] created by [link]. -pub(crate) struct SoftLink { - a: Box, - b: Box, -} - -impl Submission for SoftLink { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { a, b } = *self; - let result_a = a.execute(files); - let result_b = if result_a.as_ref().is_none_or(|result| result.success()) { - b.execute(files) - } else { - b.cancel() - }; - - Some(Box::new(LinkedCompletion { - a: result_a, - b: result_b, - })) - } - - fn cancel(self: Box) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.cancel(), - b: b.cancel(), - })) - } -} - -/// Link `a` and `b`, such that `b` gets executed after `a`. -/// -/// If `a` fails (i.e. its [Completion::success] returns `false`), `b` is -/// cancelled. -/// -/// Corresponds to io-uring's `IOSQE_IO_LINK` flag. To emulate -/// `IOSQE_IO_HARDLINK`, see [hard_link]. -pub fn link(a: Box, b: Box) -> Box { - Box::new(SoftLink { a, b }) -} - -/// [Submission] created by [hard_link]. -pub(crate) struct HardLink { - a: Box, - b: Box, -} - -impl Submission for HardLink { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.execute(files), - b: b.execute(files), - })) - } - - fn cancel(self: Box) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.cancel(), - b: b.cancel(), - })) - } -} - -/// Link `a` and `b`, such that `b` gets executed after `a`. -/// -/// Unlike [link], this executes both submissions regardless of the result. It -/// just enforces the ordering constraint that `b` will never execute before -/// `a`. -/// -/// Corresponds to io-uring's `IOSQE_IO_HARDLINK` flag. To emulate -/// `IOSQE_IO_LINK`, see [link]. -pub fn hard_link(a: Box, b: Box) -> Box { - Box::new(HardLink { a, b }) -} - -struct LinkedCompletion { - a: Option>, - b: Option>, -} - -impl Completion for LinkedCompletion { - fn success(&self) -> bool { - self.a.as_ref().is_none_or(|result| result.success()) && self.b.as_ref().is_none_or(|result| result.success()) - } - - fn complete(self: Box) { - let Self { a, b } = *self; - if let Some(a) = a { - a.complete(); - } - if let Some(b) = b { - b.complete(); - } - } -} - -/// [Submission] created by [open_file]. -pub(crate) struct OpenFile { - path: Box, - on_complete: OnComplete>, -} - -impl Submission for OpenFile { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { path, on_complete } = *self; - let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { path: _, on_complete } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [create_file]. -pub(crate) struct CreateFile { - path: Box, - len: u64, - on_complete: OnComplete>, -} - -impl Submission for CreateFile { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { path, len, on_complete } = *self; - let result = (|| { - let file = match files.entry(path.clone()) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(_) => Err(Error::FileAlreadyExists { path }), - }?; - file.set_len(len)?; - Ok(file) - })(); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { - path: _, - len: _, - on_complete, - } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [get_len]. -pub(crate) struct GetLen { - fd: fs::File, - on_complete: OnComplete>, -} - -impl Submission for GetLen { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { fd, on_complete } = *self; - let result = Ok(fd.len()); - Some(completion(true, result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { fd: _, on_complete } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [set_len]. -pub(crate) struct SetLen { - fd: fs::File, - len: u64, - on_complete: OnComplete>, -} - -impl Submission for SetLen { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { fd, len, on_complete } = *self; - let result = fd.set_len(len).map_err(Error::from); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - len: _, - on_complete, - } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -pub struct PagedOpState { - buf: Option, - on_complete: Option>>>, - remaining: usize, - first_error: Option, -} - -fn complete_page_op( - state: &Arc>>, - result: Result<(), Error>, -) -> Option>> { - let complete = { - let mut state = state.lock(); - if let Err(e) = result - && state.first_error.is_none() - { - state.first_error.replace(e); - } - assert!(state.remaining > 0); - state.remaining -= 1; - - state.remaining == 0 - }; - - complete.then(|| Box::new(PageOpCompletion { state: state.clone() })) -} - -struct PageOpCompletion { - state: Arc>>, -} - -impl Completion for PageOpCompletion { - fn success(&self) -> bool { - self.state.lock().first_error.is_none() - } - - fn complete(self: Box) { - let (on_complete, result) = { - let mut state = self.state.lock(); - - assert_eq!(state.remaining, 0); - - let buf = state.buf.take().expect("write completed more than once"); - let on_complete = state.on_complete.take().expect("write completed more than once"); - - let result = match state.first_error.take() { - None => Ok(buf), - Some(error) => Err(ErrorWith { error, with: buf }), - }; - - (on_complete, result) - }; - - let _ = on_complete.send(result); - } -} - -pub(crate) struct WritePage { - fd: fs::File, - file_page: usize, - buf_page: usize, - state: Arc>>, -} - -impl Submission for WritePage { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { - fd, - file_page, - buf_page, - state, - } = *self; - - let result = { - let state_ref = state.lock(); - let buf = state_ref.buf.as_ref().expect("buffer went away"); - - let start = buf_page * SECTOR_SIZE; - let end = start + SECTOR_SIZE; - fd.write_page(&buf.as_bytes()[start..end], file_page as _) - }; - complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - file_page: _, - buf_page: _, - state, - } = *self; - complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) - } -} - -pub(crate) struct ReadPage { - fd: fs::File, - file_page: usize, - buf_page: usize, - state: Arc>>, -} - -impl Submission for ReadPage { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { - fd, - file_page, - buf_page, - state, - } = *self; - - let result = { - let mut state_ref = state.lock(); - let buf = state_ref.buf.as_mut().expect("buffer went away"); - - let start = buf_page * SECTOR_SIZE; - let end = start + SECTOR_SIZE; - fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) - }; - complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - file_page: _, - buf_page: _, - state, - } = *self; - complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) - } -} - -#[cfg(test)] -mod tests { - use core::any::Any; - - use super::*; - - #[test] - fn downcast() { - let sqe: Box = noop(); - sqe.downcast::().unwrap(); - } -} diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 1aeb31c0af1..c49f9f690b6 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,7 +5,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO, Statx}; use static_assertions::assert_not_impl_any; use tokio::runtime; @@ -80,15 +80,13 @@ impl SpacetimeIO for TokioIO { .into() } - fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + fn create_file(&self, path: &str) -> Self::Completion> { let path = PathBuf::from(path); self.rt .spawn_blocking(move || { let mut open_options = std::fs::File::options(); open_options.read(true).write(true).create_new(true); - let file = platform::open_with_direct_io(open_options, path)?; - file.set_len(len)?; - Ok(Arc::new(file)) + platform::open_with_direct_io(open_options, path).map(Arc::new) }) .into() } @@ -140,11 +138,11 @@ impl SpacetimeIO for TokioIO { .into() } - fn length(&self, fd: Self::Fd) -> Self::Completion> { + fn statx(&self, fd: Self::Fd) -> Self::Completion> { self.rt .spawn_blocking(move || { let mut fd = fd.try_clone()?; - file_length(&mut fd) + file_length(&mut fd).map(Statx::from_size) }) .into() } From 059a6ac5454047bde10d1068aa679ff64ebbffef Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 10:06:01 +0200 Subject: [PATCH 14/19] Adjust queuing / overflow behavior to match io-uring more closely --- crates/runtime-core/src/sim/io/executor.rs | 156 +++++++++++++++------ crates/runtime-core/src/sim/io/mod.rs | 4 +- 2 files changed, 116 insertions(+), 44 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index 9135fa45c7c..a4f6eb2af70 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -5,7 +5,8 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::result::Result; +use core::{num::NonZeroUsize, result::Result}; +use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, @@ -15,9 +16,18 @@ use crate::{ }, }; +/// Dependency on the previous [Sqe]. +/// +/// A link imposes an ordering constraint: the [Sqe] carrying the link will not +/// be executed before the preceding one completed. Note that a link is only +/// meaningful within a batch of SQEs submitted together. #[derive(Clone, Copy)] pub enum LinkKind { + /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. + /// Analogous to `IOSQE_IO_LINK`. Soft, + /// Run the SQE regardless of the preceding SQE's result. + /// Analoguous to `IOSQE_IO_HARDLINK`. Hard, } @@ -447,6 +457,7 @@ enum InFlightInner { Noop, } +/* pub enum WriteFault { /// Misdirect the write to an arbitrary page offset in the file. Misdirected { page_offset: usize }, @@ -465,25 +476,86 @@ pub enum WriteFault { pub trait FaultInjector { fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; } +*/ + +/// Completion queue overflow policy. +/// +/// Note that we do **not** model `IORING_FEAT_NODROP`, because we never want +/// the application to rely on dynamic memory allocation in the kernel. +/// +/// The default is to panic, which should prompt the user to adjust queue size +/// configuration. However, sometimes it may be useful to see how the +/// application behaves when completions are dropped. +#[derive(Default)] +pub enum OnCqOverflow { + #[default] + Panic, + Drop, +} + +pub struct Options { + /// Capacity of the submission queue. + /// + /// This basically limits how many [Sqe]s can be submitted in one batch. + /// Should be a power of 2, or is otherwise rounded up to the next power of + /// 2. + pub capacity: NonZeroUsize, + /// Override the completion queue capacity. + /// + /// By default, the completion queue's capacity is twice the submission + /// queue's. This can be insufficient for some workloads, so this setting + /// can be used to override the default. + /// + /// Should be a power of 2, or is otherwise rounder up to the next power of + /// two. + pub cq_capacity: Option, + /// What to do if the completion queue overflows. + pub cq_overflow: OnCqOverflow, +} + +impl Default for Options { + fn default() -> Self { + Self { + capacity: NonZeroUsize::new(8).unwrap(), + cq_capacity: None, + cq_overflow: OnCqOverflow::default(), + } + } +} -pub struct Executor { +pub struct Executor { submissions: VecDeque>, completions: VecDeque>, - in_flight: [Option>; MAX_INFLIGHT], + in_flight: Slab>, executing: VecDeque, fstree: BTreeMap, fs::File>, -} -impl Executor { - pub fn with_capacity(capacity: usize) -> Self { + cq_overflow: OnCqOverflow, + cq_dropped: usize, +} + +impl Executor { + pub fn new( + Options { + capacity, + cq_capacity, + cq_overflow, + }: Options, + ) -> Self { + let sq_capacity = capacity.get().next_power_of_two(); + let cq_capacity = cq_capacity + .map(|c| c.get().next_power_of_two()) + .unwrap_or_else(|| 2 * sq_capacity); Self { - submissions: VecDeque::with_capacity(capacity), - completions: VecDeque::with_capacity(capacity), - in_flight: core::array::from_fn(|_| None), + submissions: VecDeque::with_capacity(sq_capacity), + completions: VecDeque::with_capacity(cq_capacity), + in_flight: Slab::new(), executing: VecDeque::new(), fstree: BTreeMap::new(), + cq_overflow, + cq_dropped: 0, } } @@ -502,13 +574,22 @@ impl Executor { } fn complete(&mut self, cqe: Cqe) { - assert!( - self.completions.len() < self.completions.capacity(), - "completion queue overflow" - ); + if self.completions.len() == self.completions.capacity() { + match self.cq_overflow { + OnCqOverflow::Panic => panic!("completion queue overflow"), + OnCqOverflow::Drop => { + self.cq_dropped += 1; + return; + } + } + } self.completions.push_back(cqe); } + pub fn dropped_completions(&self) -> usize { + self.cq_dropped + } + pub fn completed(&mut self) -> impl Iterator> { self.completions.drain(..) } @@ -522,12 +603,7 @@ impl Executor { fn schedule(&mut self) -> bool { let mut progress = false; - // Fill free execution slots. - for (id, slot) in self.in_flight.iter_mut().filter(|f| f.is_none()).enumerate() { - let Some(sqe) = self.submissions.pop_front() else { - break; - }; - + while let Some(sqe) = self.submissions.pop_front() { // If the sqe is linked, pop the whole chain. // Links of sqes not submitted in the same batch are ignored. let mut successors = VecDeque::new(); @@ -545,10 +621,10 @@ impl Executor { } } } - - let (in_flight, ops) = sqe.inner.schedule(id); + let slot = self.in_flight.vacant_entry(); + let (in_flight, ops) = sqe.inner.schedule(slot.key()); self.executing.extend(ops); - slot.replace(InFlight { + slot.insert(InFlight { inner: in_flight, blocked: successors, user_data: sqe.user_data, @@ -580,7 +656,7 @@ impl Executor { results, }, .. - } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected write") }; @@ -604,7 +680,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected write") }; @@ -634,7 +710,7 @@ impl Executor { results, }, .. - } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected read") }; @@ -658,7 +734,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invlid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected read") }; @@ -679,7 +755,7 @@ impl Executor { inner: InFlightInner::Open { sqe: Open { path } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected open") }; @@ -694,7 +770,7 @@ impl Executor { inner: InFlightInner::Create { sqe: Create { path } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected create") }; @@ -714,7 +790,7 @@ impl Executor { inner: InFlightInner::Stat { sqe: Stat { fd } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected stat") }; @@ -733,7 +809,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fallocate") }; @@ -750,7 +826,7 @@ impl Executor { inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fsync") }; @@ -770,7 +846,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fdatasync") }; @@ -786,7 +862,7 @@ impl Executor { inner: InFlightInner::Noop, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected noop") }; @@ -805,12 +881,7 @@ impl Executor { false } - fn schedule_linked( - &mut self, - in_flight_slot: SqeId, - prev_succeeded: bool, - mut blocked: VecDeque>, - ) { + fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { if let Some(Blocked { link, sqe: next, @@ -830,13 +901,14 @@ impl Executor { } } (LinkKind::Soft, true) | (LinkKind::Hard, _) => { - let (inner, ops) = next.schedule(in_flight_slot); + let (inner, ops) = next.schedule(sqe); self.executing.extend(ops); - self.in_flight[in_flight_slot].replace(InFlight { + let slot = self.in_flight.get_mut(sqe).expect("invalid sqe id"); + *slot = InFlight { inner, blocked, user_data, - }); + }; } } } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index d981ba38193..831dcacb08d 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -149,7 +149,7 @@ impl SimulatorIO { } struct SimulatorInner { - executor: spin::Mutex>, + executor: spin::Mutex>, pending: spin::Mutex>, buffers: Arc>>, } @@ -157,7 +157,7 @@ struct SimulatorInner { impl Default for SimulatorInner { fn default() -> Self { Self { - executor: spin::Mutex::new(Executor::with_capacity(128)), + executor: spin::Mutex::new(Executor::new(<_>::default())), pending: <_>::default(), buffers: <_>::default(), } From 31baac575417fdca32ec0abc229f74874d98d51e Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 11:13:29 +0200 Subject: [PATCH 15/19] Slight cleanup --- crates/runtime-core/src/sim/io/executor.rs | 2 - crates/runtime-core/src/sim/io/mod.rs | 214 ++++++++------------- 2 files changed, 77 insertions(+), 139 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index a4f6eb2af70..a9772833bef 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -457,7 +457,6 @@ enum InFlightInner { Noop, } -/* pub enum WriteFault { /// Misdirect the write to an arbitrary page offset in the file. Misdirected { page_offset: usize }, @@ -476,7 +475,6 @@ pub enum WriteFault { pub trait FaultInjector { fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; } -*/ /// Completion queue overflow policy. /// diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 831dcacb08d..f887ad6cf3e 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -146,6 +146,57 @@ impl SimulatorIO { progress } + + fn submit( + &self, + sqe: Sqe, + completion_handle: impl FnOnce(CompletionSender) -> CompletionHandle, + ) -> Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([sqe.attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(Error::SubmissionQueueOverflow)) + .unwrap_or_else(|_| unreachable!("rx is alive")), + Ok(()) => { + pending_entry.insert(completion_handle(tx)); + } + } + + rx.into() + } + + fn submit_with( + &self, + sqe: Sqe, + buf: ErasedBox, + completion_handle: impl FnOnce(CompletionSender>, usize) -> CompletionHandle, + ) -> Completion>> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([sqe.attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: buf, + })) + .unwrap_or_else(|_| unreachable!("rx is alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(buf); + pending_entry.insert(completion_handle(tx, buf_key)); + } + } + + Completion::mapped(rx, reify) + } } struct SimulatorInner { @@ -164,12 +215,14 @@ impl Default for SimulatorInner { } } +pub type CompletionReceiver = oneshot::Receiver>; + #[must_use = "completions must be polled to completion"] pub struct Completion(CompletionInner); impl Completion { pub fn mapped( - rx: oneshot::Receiver>>, + rx: CompletionReceiver>, map: fn(Result>) -> T, ) -> Self { Self(CompletionInner::Mapped { rx, map }) @@ -196,7 +249,7 @@ enum CompletionInner { rx: oneshot::Receiver, }, Mapped { - rx: oneshot::Receiver>>, + rx: CompletionReceiver>, map: fn(Result>) -> T, }, } @@ -218,37 +271,38 @@ impl Future for CompletionInner { } } +type CompletionSender = oneshot::Sender>; enum CompletionHandle { Write { - tx: oneshot::Sender>>, + tx: CompletionSender>, buf_key: usize, }, Read { - tx: oneshot::Sender>>, + tx: CompletionSender>, buf_key: usize, }, Open { - tx: oneshot::Sender>, + tx: CompletionSender, }, Create { - tx: oneshot::Sender>, + tx: CompletionSender, }, Stat { - tx: oneshot::Sender>, + tx: CompletionSender, }, Fallocate { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, Fsync { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, Fdatasync { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, // TODO: We may use this for timeouts. #[allow(unused)] Noop { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, } @@ -258,37 +312,11 @@ impl SpacetimeIO for SimulatorIO { type Completion = Completion; fn open_file(&self, path: &str) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::open(path).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Open { tx }); - } - } - - rx.into() + self.submit(Sqe::open(path), |tx| CompletionHandle::Open { tx }) } fn create_file(&self, path: &str) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::create(path).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Create { tx }); - } - } - - rx.into() + self.submit(Sqe::create(path), |tx| CompletionHandle::Create { tx }) } fn write_all_at( @@ -297,29 +325,11 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - let erased_buf = ErasedBox::from_aligned(buf); let buf_ptr = erased_buf.as_ptr(); - - match executor.submit([Sqe::write(fd, buf_ptr, offset).attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: erased_buf, - })) - .unwrap_or_else(|_| unreachable!("rx is still alive")), - Ok(()) => { - let buf_key = self.inner.buffers.lock().insert(erased_buf); - pending_entry.insert(CompletionHandle::Write { tx, buf_key }); - } - } - - Completion::mapped(rx, reify) + self.submit_with(Sqe::write(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { + CompletionHandle::Write { tx, buf_key } + }) } fn read_exact_at( @@ -328,97 +338,27 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - let erased_buf = ErasedBox::from_aligned(buf); let buf_ptr = erased_buf.as_ptr(); - - match executor.submit([Sqe::read(fd, buf_ptr, offset).attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: erased_buf, - })) - .unwrap_or_else(|_| unreachable!("rx is still alive")), - Ok(()) => { - let buf_key = self.inner.buffers.lock().insert(erased_buf); - pending_entry.insert(CompletionHandle::Read { tx, buf_key }); - } - } - - Completion::mapped(rx, reify) + self.submit_with(Sqe::read(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { + CompletionHandle::Read { tx, buf_key } + }) } fn fsync(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fsync(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fsync { tx }); - } - } - - rx.into() + self.submit(Sqe::fsync(fd), |tx| CompletionHandle::Fsync { tx }) } fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fdatasync(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fdatasync { tx }); - } - } - - rx.into() + self.submit(Sqe::fdatasync(fd), |tx| CompletionHandle::Fdatasync { tx }) } fn reserve(&self, fd: Self::Fd, total_size: u64) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fallocate(fd, total_size).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fallocate { tx }); - } - } - - rx.into() + self.submit(Sqe::fallocate(fd, total_size), |tx| CompletionHandle::Fallocate { tx }) } fn statx(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::stat(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Stat { tx }); - } - } - - rx.into() + self.submit(Sqe::stat(fd), |tx| CompletionHandle::Stat { tx }) } } From 80f2787521ea7c3568db149a554afb2a72cb7e5e Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 14:54:13 +0200 Subject: [PATCH 16/19] Model file durability --- crates/runtime-core/src/sim/io/executor.rs | 500 +++++++++++++-------- crates/runtime-core/src/sim/io/fs.rs | 152 +++++-- 2 files changed, 422 insertions(+), 230 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index a9772833bef..d2e34404dc0 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -5,13 +5,16 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::{num::NonZeroUsize, result::Result}; +use core::{mem, num::NonZeroUsize, result::Result}; use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, sim::{ - io::{fs, Error, Instant}, + io::{ + fs::{self, Datasync}, + Error, Instant, + }, Rng, }, }; @@ -221,14 +224,48 @@ impl SqeInner { InFlightInner::Fallocate { sqe }, alloc::vec![Operation::Fallocate { sqe: sqe_id }], ), - SqeInner::Fsync(sqe) => ( - InFlightInner::Fsync { sqe }, - alloc::vec![Operation::Fsync { sqe: sqe_id }], - ), - SqeInner::Fdatasync(sqe) => ( - InFlightInner::Fdatasync { sqe }, - alloc::vec![Operation::Fdatasync { sqe: sqe_id }], - ), + SqeInner::Fsync(sqe) => { + let Fsync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Sector(offset), + }) + .chain([Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Length, + }]) + .collect::>(); + let in_flight = InFlightInner::Fsync { + sqe, + op_count: ops.len(), + }; + + (in_flight, ops) + } + SqeInner::Fdatasync(sqe) => { + let Fdatasync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Sector(offset), + }) + .chain([Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Length, + }]) + .collect::>(); + let in_flight = InFlightInner::Fdatasync { + sqe, + op_count: ops.len(), + }; + + (in_flight, ops) + } SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), } } @@ -379,6 +416,12 @@ impl Cqe { type SqeId = usize; +// TODO: There is no difference between fsync and fdatasync as long as we don't +// have an API to fsync the directory of a file after it was created. +enum FsyncEffect { + Datasync(Datasync), +} + enum Operation { WriteSector { sqe: SqeId, @@ -404,9 +447,11 @@ enum Operation { }, Fsync { sqe: SqeId, + effect: FsyncEffect, }, Fdatasync { sqe: SqeId, + effect: Datasync, }, Noop { sqe: SqeId, @@ -450,9 +495,11 @@ enum InFlightInner { }, Fsync { sqe: Fsync, + op_count: usize, }, Fdatasync { sqe: Fdatasync, + op_count: usize, }, Noop, } @@ -484,7 +531,7 @@ pub trait FaultInjector { /// The default is to panic, which should prompt the user to adjust queue size /// configuration. However, sometimes it may be useful to see how the /// application behaves when completions are dropped. -#[derive(Default)] +#[derive(Clone, Copy, Default)] pub enum OnCqOverflow { #[default] Panic, @@ -557,6 +604,43 @@ impl Executor { } } + /// Simulate a power-loss crash. + /// + /// All submitted and executing operations are cancelled, and files reset to + /// their durable state. After this method returns, the completion queue is + /// empty. + pub fn crash(&mut self) { + self.submissions.clear(); + self.completions.clear(); + self.in_flight.clear(); + self.executing.clear(); + self.cq_dropped = 0; + + for file in self.fstree.values_mut() { + file.crash(); + } + } + + /// Restart the executor, simulating a process crash. + /// + /// Unlike [Self::crash], this will drive the currently executing operations + /// to completion. Submissions that were not yet scheduled are dropped. The + /// file state remains unchanged. + /// + /// After this method returns, the completion queue is empty. + pub fn restart(&mut self, now: Instant) { + self.submissions.clear(); + let cq_overflow_orig = self.cq_overflow; + self.cq_overflow = OnCqOverflow::Drop; + let executing = mem::take(&mut self.executing); + for op in executing { + self.execute(op, now); + } + self.completions.clear(); + self.cq_overflow = cq_overflow_orig; + self.cq_dropped = 0; + } + pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> where Batch: IntoIterator>, @@ -594,7 +678,7 @@ impl Executor { pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { let mut progress = self.schedule(); - progress |= self.execute(rng, now); + progress |= self.execute_random(rng, now); progress } @@ -634,249 +718,275 @@ impl Executor { progress } - fn execute(&mut self, rng: &Rng, now: Instant) -> bool { + fn execute_random(&mut self, rng: &Rng, now: Instant) -> bool { if self.executing.is_empty() { return false; } if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { - match op { - Operation::WriteSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected write") - }; - let bytes = buf.as_bytes(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &buf.as_bytes()[buf_offset..end]; - let result = fd.write_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected write") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Write { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::ReadSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected read") - }; - let bytes = buf.as_bytes_mut(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &mut buf.as_bytes_mut()[buf_offset..end]; - let result = fd.read_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; + self.execute(op, now); + true + } else { + false + } + } - if is_complete { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected read") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Read { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::Open { sqe } => { + fn execute(&mut self, op: Operation, _now: Instant) { + match op { + Operation::WriteSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { let InFlight { - inner: InFlightInner::Open { sqe: Open { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Write { + sqe: Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { - unreachable!("invalid sqe: expected open") + unreachable!("invalid sqe: expected write") }; + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); - let is_success = result.is_ok(); - self.complete(Cqe::Open { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Create { sqe } => { + let buf = &buf.as_bytes()[buf_offset..end]; + let result = fd.write_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { let InFlight { - inner: InFlightInner::Create { sqe: Create { path } }, + inner: + InFlightInner::Write { + sqe: Write { mut buf, .. }, + op_count, + results, + }, blocked, user_data, } = self.in_flight.remove(sqe) else { - unreachable!("invalid sqe: expected create") + unreachable!("invalid sqe: expected write") }; - - let result = match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { - path: entry.key().clone(), - }), + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), }; let is_success = result.is_ok(); - self.complete(Cqe::Create { result, user_data }); + self.complete(Cqe::Write { result, user_data }); self.schedule_linked(sqe, is_success, blocked); } - Operation::Stat { sqe } => { + } + Operation::ReadSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { let InFlight { - inner: InFlightInner::Stat { sqe: Stat { fd } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Read { + sqe: Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { - unreachable!("invalid sqe: expected stat") + unreachable!("invalid sqe: expected read") }; + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - self.complete(Cqe::Stat { - result: Ok(Statx { size: fd.len() }), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Fallocate { sqe } => { + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + let result = fd.read_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { let InFlight { inner: - InFlightInner::Fallocate { - sqe: Fallocate { fd, total_len }, + InFlightInner::Read { + sqe: Read { mut buf, .. }, + op_count, + results, }, blocked, user_data, } = self.in_flight.remove(sqe) else { - unreachable!("invalid sqe: expected fallocate") + unreachable!("invalid sqe: expected read") }; - - self.complete(Cqe::Fallocate { - result: fd.set_len(total_len).map_err(Error::from), - user_data, - }); - self.schedule_linked(sqe, true, blocked); + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); } - Operation::Fsync { sqe } => { - // TODO: Do something fallible with fd. + } + Operation::Open { sqe } => { + let InFlight { + inner: InFlightInner::Open { sqe: Open { path } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected open") + }; + + let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Create { sqe } => { + let InFlight { + inner: InFlightInner::Create { sqe: Create { path } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected create") + }; + + let result = match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Stat { sqe } => { + let InFlight { + inner: InFlightInner::Stat { sqe: Stat { fd } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected stat") + }; + + self.complete(Cqe::Stat { + result: Ok(Statx { size: fd.len() }), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fallocate { sqe } => { + let InFlight { + inner: + InFlightInner::Fallocate { + sqe: Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected fallocate") + }; + + self.complete(Cqe::Fallocate { + result: fd.set_len(total_len).map_err(Error::from), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fsync { sqe, effect } => { + let is_complete = { let InFlight { - inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Fsync { + sqe: Fsync { fd }, + op_count, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fsync") }; + match effect { + FsyncEffect::Datasync(effect) => fd.fdatasync([effect]), + } + *op_count -= 1; + + *op_count == 0 + }; + + if is_complete { + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); self.complete(Cqe::Fsync { result: Ok(()), user_data, }); self.schedule_linked(sqe, true, blocked); } - Operation::Fdatasync { sqe } => { - // TODO: Do something fallible with fd. + } + Operation::Fdatasync { sqe, effect } => { + let is_complete = { let InFlight { inner: InFlightInner::Fdatasync { - sqe: Fdatasync { fd: _ }, + sqe: Fdatasync { fd }, + op_count, }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fdatasync") }; - self.complete(Cqe::Fdatasync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Noop { sqe } => { - let InFlight { - inner: InFlightInner::Noop, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected noop") - }; + fd.fdatasync([effect]); + *op_count -= 1; + + *op_count == 0 + }; - self.complete(Cqe::Noop { + if is_complete { + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + self.complete(Cqe::Fdatasync { result: Ok(()), user_data, }); self.schedule_linked(sqe, true, blocked); } } + Operation::Noop { sqe } => { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected noop") + }; - return true; + self.complete(Cqe::Noop { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } } - - false } fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 93e1288fac3..64aadf5a017 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,6 +1,6 @@ use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cmp, fmt, + fmt, sync::atomic::{AtomicU64, Ordering}, }; @@ -41,6 +41,75 @@ impl Page { } } +#[derive(Default)] +struct PageMap { + volatile: BTreeMap>, + durable: BTreeMap>, +} + +impl PageMap { + /// Reset the volatile to the durable state. + fn crash(&mut self) { + self.volatile = self.durable.clone(); + } + + /// Move the page at `index` from the volatile to the durable state. + fn sync(&mut self, index: PageIndex) { + self.durable.insert(index, self.volatile.get(&index).cloned().unwrap()); + } + + /// Get the page at `index` for reading. Uses the durable state. + fn get_page(&self, index: PageIndex) -> Option> { + self.durable.get(&index).cloned() + } + + /// Get the page at `index` for writing, or allocate a new page. + /// Uses the volatile state. + fn get_or_allocate_page(&mut self, index: PageIndex) -> Arc { + Arc::clone(self.volatile.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) + } + + /// Change the allocated space, allocating or deallocating pages as needed. + /// Changes the volatile state only. + fn set_len_volatile(&mut self, new_len: u64) { + Self::set_len(&mut self.volatile, new_len); + } + + /// Like [Self::set_len], but operate on the durable state only. + fn set_len_durable(&mut self, new_len: u64) { + Self::set_len(&mut self.durable, new_len); + } + + fn set_len(page_map: &mut BTreeMap>, new_len: u64) { + use core::cmp::Ordering::*; + + let old_len = page_map.len() as u64; + match new_len.cmp(&old_len) { + Equal => {} + Greater => { + let first_new_page = old_len / PAGE_SIZE_U64; + let end_page = new_len / PAGE_SIZE_U64; + + for index in first_new_page..end_page { + page_map + .entry(PageIndex(index)) + .or_insert_with(|| Arc::new(Page::zeroed())); + } + } + Less => { + let first_removed = PageIndex::from_offset(new_len); + let removed = page_map.split_off(&first_removed); + drop(removed); + } + } + } +} + +pub enum Datasync { + Sector(u64), + Length, +} + /// A memory-backed file. /// /// A [File] is backed by a sparse array of [Page]s. Missing pages are read as @@ -50,26 +119,39 @@ impl Page { /// or written. Writing a page is atomic. #[derive(Clone)] pub struct File { - pages: Arc>>>, - len: Arc, + pages: Arc>, + + volatile_len: Arc, + durable_len: Arc, } impl fmt::Debug for File { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("File").field("len", &self.len).finish() + f.debug_struct("File") + .field("volatile_len", &self.volatile_len) + .field("durable_len", &self.durable_len) + .finish() } } impl File { pub(super) fn new() -> Self { Self { - pages: Arc::new(spin::Mutex::new(BTreeMap::new())), - len: Arc::new(AtomicU64::new(0)), + pages: <_>::default(), + volatile_len: <_>::default(), + durable_len: <_>::default(), } } + /// Simulate a crash by resetting to the durable state. + pub(super) fn crash(&self) { + self.volatile_len + .store(self.durable_len.load(Ordering::Relaxed), Ordering::Relaxed); + self.pages.lock().crash(); + } + pub(super) fn len(&self) -> u64 { - self.len.load(Ordering::Relaxed) + self.volatile_len.load(Ordering::Relaxed) } #[allow(unused)] @@ -84,33 +166,11 @@ impl File { /// Extending allocates pages eagerly as needed. Shrinking drops all pages /// at or beyond the new EOF. pub(super) fn set_len(&self, new_len: u64) -> Result<()> { - use cmp::Ordering::*; - if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } - let old_len = self.len(); - - match new_len.cmp(&old_len) { - Equal => {} - Greater => { - let first_new_page = old_len / PAGE_SIZE_U64; - let end_page = new_len / PAGE_SIZE_U64; - - for index in first_new_page..end_page { - self.get_or_allocate_page(PageIndex(index)); - } - - self.len.store(new_len, Ordering::Relaxed); - } - Less => { - self.len.store(new_len, Ordering::Relaxed); - - let first_removed = PageIndex::from_offset(new_len); - let removed = self.pages.lock().split_off(&first_removed); - drop(removed); - } - } + self.pages.lock().set_len_volatile(new_len); + self.volatile_len.store(new_len, Ordering::Relaxed); Ok(()) } @@ -147,17 +207,39 @@ impl File { .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) .ok_or(Error::OffsetOverflow)?; - self.len.fetch_max(end, Ordering::Relaxed); + self.volatile_len.fetch_max(end, Ordering::Relaxed); Ok(()) } + /// Execute an `fdatasync(2)` operation as a series of [Datasync] effects. + /// + /// The result may or may not leave the durable state in the same state as + /// the volatile state at the time the operation started. + /// + /// It is the caller's responsibility to decide whether the operation is + /// considered successful - a partial operation may report success, or a + /// complete operation may report failure. + pub(super) fn fdatasync(&self, ops: impl IntoIterator) { + for op in ops { + match op { + Datasync::Sector(offset) => { + self.pages.lock().sync(PageIndex(offset)); + } + Datasync::Length => { + let new_durable_len = self.volatile_len.load(Ordering::Relaxed); + self.durable_len.store(new_durable_len, Ordering::Relaxed); + self.pages.lock().set_len_durable(new_durable_len); + } + } + } + } + fn get_page(&self, index: PageIndex) -> Option> { - self.pages.lock().get(&index).cloned() + self.pages.lock().get_page(index) } fn get_or_allocate_page(&self, index: PageIndex) -> Arc { - let mut pages = self.pages.lock(); - Arc::clone(pages.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) + self.pages.lock().get_or_allocate_page(index) } } From 77b3a0bdd87f19e701b9c660d82ff850d805176a Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 3 Sep 2026 11:24:25 +0200 Subject: [PATCH 17/19] Prepare `SqeId` for export --- crates/runtime-core/src/sim/io/executor.rs | 39 +++++++++++++--------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index d2e34404dc0..e9befc20afb 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -414,7 +414,14 @@ impl Cqe { } } -type SqeId = usize; +#[derive(Clone, Copy)] +pub struct SqeId(usize); + +impl SqeId { + fn key(&self) -> usize { + self.0 + } +} // TODO: There is no difference between fsync and fdatasync as long as we don't // have an API to fsync the directory of a file after it was created. @@ -704,7 +711,7 @@ impl Executor { } } let slot = self.in_flight.vacant_entry(); - let (in_flight, ops) = sqe.inner.schedule(slot.key()); + let (in_flight, ops) = sqe.inner.schedule(SqeId(slot.key())); self.executing.extend(ops); slot.insert(InFlight { inner: in_flight, @@ -746,7 +753,7 @@ impl Executor { results, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected write") }; @@ -770,7 +777,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected write") }; @@ -800,7 +807,7 @@ impl Executor { results, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected read") }; @@ -824,7 +831,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected read") }; @@ -845,7 +852,7 @@ impl Executor { inner: InFlightInner::Open { sqe: Open { path } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected open") }; @@ -860,7 +867,7 @@ impl Executor { inner: InFlightInner::Create { sqe: Create { path } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected create") }; @@ -880,7 +887,7 @@ impl Executor { inner: InFlightInner::Stat { sqe: Stat { fd } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected stat") }; @@ -899,7 +906,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected fallocate") }; @@ -919,7 +926,7 @@ impl Executor { op_count, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fsync") }; @@ -933,7 +940,7 @@ impl Executor { }; if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); self.complete(Cqe::Fsync { result: Ok(()), user_data, @@ -950,7 +957,7 @@ impl Executor { op_count, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fdatasync") }; @@ -962,7 +969,7 @@ impl Executor { }; if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); self.complete(Cqe::Fdatasync { result: Ok(()), user_data, @@ -975,7 +982,7 @@ impl Executor { inner: InFlightInner::Noop, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected noop") }; @@ -1011,7 +1018,7 @@ impl Executor { (LinkKind::Soft, true) | (LinkKind::Hard, _) => { let (inner, ops) = next.schedule(sqe); self.executing.extend(ops); - let slot = self.in_flight.get_mut(sqe).expect("invalid sqe id"); + let slot = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id"); *slot = InFlight { inner, blocked, From c5c796327a4e08b7563b1cf7e6ad7e685436f89c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 3 Sep 2026 17:34:07 +0200 Subject: [PATCH 18/19] WIP: fault injection API --- crates/runtime-core/src/sim/io/executor.rs | 1181 +++++++---------- .../runtime-core/src/sim/io/executor/sqe.rs | 391 ++++++ crates/runtime-core/src/sim/io/fs.rs | 1 + crates/runtime-core/src/sim/io/mod.rs | 12 +- 4 files changed, 903 insertions(+), 682 deletions(-) create mode 100644 crates/runtime-core/src/sim/io/executor/sqe.rs diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index e9befc20afb..9d4237a760b 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -11,351 +11,47 @@ use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, sim::{ - io::{ - fs::{self, Datasync}, - Error, Instant, - }, + io::{fs, Error, Instant}, Rng, }, }; -/// Dependency on the previous [Sqe]. -/// -/// A link imposes an ordering constraint: the [Sqe] carrying the link will not -/// be executed before the preceding one completed. Note that a link is only -/// meaningful within a batch of SQEs submitted together. -#[derive(Clone, Copy)] -pub enum LinkKind { - /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. - /// Analogous to `IOSQE_IO_LINK`. - Soft, - /// Run the SQE regardless of the preceding SQE's result. - /// Analoguous to `IOSQE_IO_HARDLINK`. - Hard, -} - -pub struct Sqe { - inner: SqeInner, - link: Option, - user_data: Option, -} - -impl Sqe { - pub fn link(mut self, kind: Option) -> Self { - self.link = kind; - self - } - - pub fn is_linked(&self) -> bool { - self.link.is_some() - } - - pub fn attach(mut self, user_data: T) -> Self { - self.user_data.replace(user_data); - self - } - - pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { - Write { fd, buf, offset }.into() - } - - pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { - Read { fd, buf, offset }.into() - } +pub use crate::sim::io::fs::Datasync; - pub fn open(path: impl AsRef) -> Self { - Open { - path: path.as_ref().into(), - } - .into() - } - - pub fn create(path: impl AsRef) -> Self { - Create { - path: path.as_ref().into(), - } - .into() - } - - pub fn stat(fd: fs::File) -> Self { - Stat { fd }.into() - } - - pub fn fallocate(fd: fs::File, len: u64) -> Self { - Fallocate { fd, total_len: len }.into() - } - - pub fn fsync(fd: fs::File) -> Self { - Fsync { fd }.into() - } - - pub fn fdatasync(fd: fs::File) -> Self { - Fdatasync { fd }.into() - } - - pub fn noop() -> Self { - SqeInner::Noop.into() - } -} +mod sqe; +use sqe::SqeInner; +pub use sqe::{LinkKind, Sqe, SqeId}; -impl> From for Sqe { - fn from(inner: U) -> Self { - Self { - inner: inner.into(), - link: None, - user_data: None, - } - } +// TODO: There is no difference between fsync and fdatasync as long as we don't +// have an API to fsync the directory of a file after it was created. +#[derive(Clone, Copy)] +pub enum FsyncEffect { + Datasync(Datasync), } -enum SqeInner { - Write(Write), - Read(Read), - Open(Open), - Create(Create), - Stat(Stat), - Fallocate(Fallocate), - Fsync(Fsync), - Fdatasync(Fdatasync), +#[derive(Clone, Copy)] +pub enum Operation { + WriteSector(WriteSector), + ReadSector(ReadSector), + Open, + Create, + Stat, + Fallocate, + Fsync { effect: FsyncEffect }, + Fdatasync { effect: Datasync }, Noop, } -impl SqeInner { - fn cancel(self, user_data: Option) -> Cqe { - match self { - SqeInner::Write(..) => Cqe::Write { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Read(..) => Cqe::Read { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Open(..) => Cqe::Open { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Create(..) => Cqe::Create { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Stat(..) => Cqe::Stat { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fallocate(..) => Cqe::Fallocate { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fsync(..) => Cqe::Fsync { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fdatasync(..) => Cqe::Fdatasync { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Noop => Cqe::Noop { - result: Err(Error::Cancelled), - user_data, - }, - } - } - - fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { - match self { - SqeInner::Write(mut sqe) => { - let Write { buf, offset, .. } = &mut sqe; - let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; - - let ops = (0..page_count) - .map(|page| Operation::WriteSector { - sqe: sqe_id, - page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), - }) - .collect::>(); - let op_count = ops.len(); - let write = InFlightInner::Write { - sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (write, ops) - } - SqeInner::Read(mut sqe) => { - let Read { buf, offset, .. } = &mut sqe; - let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; - - let ops = (0..page_count) - .map(|page| Operation::ReadSector { - sqe: sqe_id, - page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), - }) - .collect::>(); - let op_count = ops.len(); - let read = InFlightInner::Read { - sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (read, ops) - } - SqeInner::Open(sqe) => ( - InFlightInner::Open { sqe }, - alloc::vec![Operation::Open { sqe: sqe_id }], - ), - SqeInner::Create(sqe) => ( - InFlightInner::Create { sqe }, - alloc::vec![Operation::Create { sqe: sqe_id }], - ), - SqeInner::Stat(sqe) => ( - InFlightInner::Stat { sqe }, - alloc::vec![Operation::Stat { sqe: sqe_id }], - ), - SqeInner::Fallocate(sqe) => ( - InFlightInner::Fallocate { sqe }, - alloc::vec![Operation::Fallocate { sqe: sqe_id }], - ), - SqeInner::Fsync(sqe) => { - let Fsync { fd } = &sqe; - - let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Sector(offset), - }) - .chain([Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Length, - }]) - .collect::>(); - let in_flight = InFlightInner::Fsync { - sqe, - op_count: ops.len(), - }; - - (in_flight, ops) - } - SqeInner::Fdatasync(sqe) => { - let Fdatasync { fd } = &sqe; - - let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Sector(offset), - }) - .chain([Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Length, - }]) - .collect::>(); - let in_flight = InFlightInner::Fdatasync { - sqe, - op_count: ops.len(), - }; - - (in_flight, ops) - } - SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), - } - } -} - -impl From for SqeInner { - fn from(inner: Write) -> Self { - Self::Write(inner) - } -} - -impl From for SqeInner { - fn from(inner: Read) -> Self { - Self::Read(inner) - } -} - -impl From for SqeInner { - fn from(inner: Open) -> Self { - Self::Open(inner) - } -} - -impl From for SqeInner { - fn from(inner: Create) -> Self { - Self::Create(inner) - } -} - -impl From for SqeInner { - fn from(inner: Stat) -> Self { - Self::Stat(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fallocate) -> Self { - Self::Fallocate(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fsync) -> Self { - Self::Fsync(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fdatasync) -> Self { - Self::Fdatasync(inner) - } -} - -struct Write { - fd: fs::File, - buf: ErasedBoxPtr, - offset: u64, -} - -struct Read { - fd: fs::File, - buf: ErasedBoxPtr, - offset: u64, -} - -struct Open { - path: Box, -} - -struct Create { - path: Box, -} - -struct Stat { - fd: fs::File, -} - -struct Fallocate { - fd: fs::File, - total_len: u64, -} - -struct Fsync { - #[allow(unused)] - fd: fs::File, +#[derive(Clone, Copy)] +pub struct WriteSector { + pub page_offset: usize, + pub buf_offset: usize, } -struct Fdatasync { - #[allow(unused)] - fd: fs::File, +#[derive(Clone, Copy)] +pub struct ReadSector { + pub page_offset: usize, + pub buf_offset: usize, } #[derive(Debug)] @@ -414,122 +110,168 @@ impl Cqe { } } -#[derive(Clone, Copy)] -pub struct SqeId(usize); - -impl SqeId { - fn key(&self) -> usize { - self.0 - } +pub struct Blocked { + pub link: LinkKind, + pub sqe: SqeInner, + pub user_data: Option, } -// TODO: There is no difference between fsync and fdatasync as long as we don't -// have an API to fsync the directory of a file after it was created. -enum FsyncEffect { - Datasync(Datasync), +pub struct InFlight { + pub inner: InFlightInner, + pub blocked: VecDeque>, + pub user_data: Option, } -enum Operation { - WriteSector { - sqe: SqeId, - page_offset: usize, - buf_offset: usize, - }, - ReadSector { - sqe: SqeId, - page_offset: usize, - buf_offset: usize, - }, - Open { - sqe: SqeId, - }, - Create { - sqe: SqeId, - }, - Stat { - sqe: SqeId, - }, - Fallocate { - sqe: SqeId, - }, - Fsync { - sqe: SqeId, - effect: FsyncEffect, - }, - Fdatasync { - sqe: SqeId, - effect: Datasync, - }, - Noop { - sqe: SqeId, - }, -} - -struct Blocked { - link: LinkKind, - sqe: SqeInner, - user_data: Option, -} - -struct InFlight { - inner: InFlightInner, - blocked: VecDeque>, - user_data: Option, -} - -enum InFlightInner { +pub enum InFlightInner { Write { - sqe: Write, + sqe: sqe::Write, op_count: usize, - results: Vec>, + results: Vec>, }, Read { - sqe: Read, + sqe: sqe::Read, op_count: usize, - results: Vec>, + results: Vec>, }, Open { - sqe: Open, + sqe: sqe::Open, }, Create { - sqe: Create, + sqe: sqe::Create, }, Stat { - sqe: Stat, + sqe: sqe::Stat, }, Fallocate { - sqe: Fallocate, + sqe: sqe::Fallocate, }, Fsync { - sqe: Fsync, + sqe: sqe::Fsync, op_count: usize, + results: Vec>, }, Fdatasync { - sqe: Fdatasync, + sqe: sqe::Fdatasync, op_count: usize, + results: Vec>, }, Noop, } -pub enum WriteFault { - /// Misdirect the write to an arbitrary page offset in the file. - Misdirected { page_offset: usize }, - /// Report the write as successful, but don't write anything. - Lost, - /// Report the write as successful, but write less bytes than requested. - Short { write_bytes: usize }, - /// Delay the write until at least `deadline`. - Delayed { deadline: Instant }, - /// Execute the side effects, but never report completion. - NoCompletion, - /// Report an error without executing side effects. - Error(Error), +struct Executing { + sqe: SqeId, + inner: Operation, +} + +impl Executing { + fn traverse(self, f: impl FnOnce(SqeId, Operation) -> Option) -> Option { + let Self { sqe, inner } = self; + f(sqe, inner).map(|inner| Self { sqe, inner }) + } +} + +pub enum Fault { + /// Drop the operation entirely. + Skip, + /// Put the operation back onto the queue for later execution. + Delay(T), + /// Execute a visible effect. + Visible(Effect), +} + +impl Fault { + fn exec_visible(self, f: impl FnOnce(EitherOrBoth)) -> Option { + match self { + Fault::Skip => None, + Fault::Delay(effect) => Some(effect), + Fault::Visible(visible) => { + visible.exec(f); + None + } + } + } +} + +pub enum Effect { + /// Run the operation as normal. + Run(T), + /// Run the effect, but report an injected error. + RunThenError { effect: T, error: Error }, + /// Skip the effect, but report an injected error. + SkipThenError { error: Error }, } -pub trait FaultInjector { - fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; +impl Effect { + fn exec(self, f: impl FnOnce(EitherOrBoth)) { + use EitherOrBoth::*; + match self { + Effect::Run(effect) => f(Left(effect)), + Effect::RunThenError { effect, error } => f(Both(effect, error)), + Effect::SkipThenError { error } => f(Right(error)), + } + } +} + +enum EitherOrBoth { + Left(T), + Right(U), + Both(T, U), +} + +impl EitherOrBoth { + fn traverse(self, f: impl FnOnce(T) -> V, g: impl FnOnce(U) -> V) -> V { + match self { + Self::Left(t) => f(t), + Self::Right(u) => g(u), + Self::Both(t, u) => { + f(t); + g(u) + } + } + } +} + +pub trait FaultInjector { + fn inject_write_sector_fault(&mut self, sqe: &InFlight, op: WriteSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_read_sector_fault(&mut self, sqe: &InFlight, op: ReadSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_open_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_create_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_stat_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fallocate_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fsync_fault(&mut self, sqe: &InFlight, op: FsyncEffect) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_fdatasync_fault(&mut self, sqe: &InFlight, op: Datasync) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_noop_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } } +pub struct NoFaults; +impl FaultInjector for NoFaults {} + /// Completion queue overflow policy. /// /// Note that we do **not** model `IORING_FEAT_NODROP`, because we never want @@ -580,7 +322,7 @@ pub struct Executor { completions: VecDeque>, in_flight: Slab>, - executing: VecDeque, + executing: VecDeque, fstree: BTreeMap, fs::File>, @@ -634,20 +376,24 @@ impl Executor { /// to completion. Submissions that were not yet scheduled are dropped. The /// file state remains unchanged. /// + /// Execution is subject to `faults`. If a fault evaluates to [Fault::Skip], + /// that operation is dropped. + /// /// After this method returns, the completion queue is empty. - pub fn restart(&mut self, now: Instant) { + pub fn restart(&mut self, faults: &mut impl FaultInjector) { self.submissions.clear(); let cq_overflow_orig = self.cq_overflow; self.cq_overflow = OnCqOverflow::Drop; let executing = mem::take(&mut self.executing); for op in executing { - self.execute(op, now); + self.execute(op, faults); } self.completions.clear(); self.cq_overflow = cq_overflow_orig; self.cq_dropped = 0; } + /// Submit a batch of [Sqe]s for later execution. pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> where Batch: IntoIterator>, @@ -675,17 +421,26 @@ impl Executor { self.completions.push_back(cqe); } + /// Number of completions that were dropped due to completion queue overflow + /// over the lifetime of this executor. + /// + /// Always zero if the executor was configured with [OnCqOverflow::Panic]. pub fn dropped_completions(&self) -> usize { self.cq_dropped } + /// Drain the completion queue. pub fn completed(&mut self) -> impl Iterator> { self.completions.drain(..) } - pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { + /// Drain the submission queue and advance one scheduled operation. + /// + /// The operation to advance is chosen randomly using `rng`. + /// The operation is subject to `faults`. + pub fn tick(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { let mut progress = self.schedule(); - progress |= self.execute_random(rng, now); + progress |= self.execute_random(rng, faults); progress } @@ -725,277 +480,353 @@ impl Executor { progress } - fn execute_random(&mut self, rng: &Rng, now: Instant) -> bool { + fn execute_random(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { if self.executing.is_empty() { return false; } if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { - self.execute(op, now); + if let Some(delay) = self.execute(op, faults) { + self.executing.push_back(delay); + } true } else { false } } - fn execute(&mut self, op: Operation, _now: Instant) { - match op { - Operation::WriteSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected write") - }; - let bytes = buf.as_bytes(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &buf.as_bytes()[buf_offset..end]; - let result = fd.write_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected write") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Write { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::ReadSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected read") - }; - let bytes = buf.as_bytes_mut(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &mut buf.as_bytes_mut()[buf_offset..end]; - let result = fd.read_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected read") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Read { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::Open { sqe } => { - let InFlight { - inner: InFlightInner::Open { sqe: Open { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected open") - }; - - let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); - let is_success = result.is_ok(); - self.complete(Cqe::Open { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Create { sqe } => { - let InFlight { - inner: InFlightInner::Create { sqe: Create { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected create") - }; - - let result = match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { - path: entry.key().clone(), - }), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Create { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Stat { sqe } => { - let InFlight { - inner: InFlightInner::Stat { sqe: Stat { fd } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected stat") - }; - - self.complete(Cqe::Stat { - result: Ok(Statx { size: fd.len() }), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Fallocate { sqe } => { - let InFlight { - inner: - InFlightInner::Fallocate { - sqe: Fallocate { fd, total_len }, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected fallocate") - }; - - self.complete(Cqe::Fallocate { - result: fd.set_len(total_len).map_err(Error::from), - user_data, - }); - self.schedule_linked(sqe, true, blocked); + fn execute(&mut self, op: Executing, faults: &mut impl FaultInjector) -> Option { + op.traverse(|sqe, op| { + let in_flight = self.in_flight.get(sqe.key()).expect("invalid sqe id"); + match op { + Operation::WriteSector(effect) => faults + .inject_write_sector_fault(in_flight, effect) + .exec_visible(|eff| self.execute_write_sector(sqe, eff)) + .map(Operation::WriteSector), + Operation::ReadSector(effect) => faults + .inject_read_sector_fault(in_flight, effect) + .exec_visible(|eff| self.execute_read_sector(sqe, eff)) + .map(Operation::ReadSector), + Operation::Open => faults + .inject_open_fault(in_flight) + .exec_visible(|eff| self.execute_open(sqe, eff)) + .map(|()| Operation::Open), + Operation::Create => faults + .inject_create_fault(in_flight) + .exec_visible(|eff| self.execute_create(sqe, eff)) + .map(|()| Operation::Create), + Operation::Stat => faults + .inject_stat_fault(in_flight) + .exec_visible(|eff| self.execute_stat(sqe, eff)) + .map(|()| Operation::Stat), + Operation::Fallocate => faults + .inject_fallocate_fault(in_flight) + .exec_visible(|eff| self.execute_fallocate(sqe, eff)) + .map(|()| Operation::Fallocate), + Operation::Fsync { effect } => faults + .inject_fsync_fault(in_flight, effect) + .exec_visible(|eff| self.execute_fsync(sqe, eff)) + .map(|effect| Operation::Fsync { effect }), + Operation::Fdatasync { effect } => faults + .inject_fdatasync_fault(in_flight, effect) + .exec_visible(|eff| self.execute_fdatasync(sqe, eff)) + .map(|effect| Operation::Fdatasync { effect }), + Operation::Noop => faults + .inject_noop_fault(in_flight) + .exec_visible(|eff| self.execute_noop(sqe, eff)) + .map(|()| Operation::Noop), } - Operation::Fsync { sqe, effect } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Fsync { - sqe: Fsync { fd }, - op_count, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected fsync") - }; - - match effect { - FsyncEffect::Datasync(effect) => fd.fdatasync([effect]), - } - *op_count -= 1; - - *op_count == 0 - }; + }) + } + + fn execute_write_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Write { + sqe: sqe::Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + let mut run = |WriteSector { + page_offset, + buf_offset, + }| { + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &buf.as_bytes()[buf_offset..end]; + fd.write_page(buf, page_offset as _).map_err(Into::into) + }; + results.push(eff.traverse(run, Err)); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Write { + sqe: sqe::Write { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected write") + }; + assert!(results.len() == op_count); + let bytes_written = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + // TODO: Propagate all errors? + let result = match results.into_iter().find_map(Result::err) { + Some(error) => Err(error), + None => Ok(bytes_written), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Write { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } - if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); - self.complete(Cqe::Fsync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - } - Operation::Fdatasync { sqe, effect } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Fdatasync { - sqe: Fdatasync { fd }, - op_count, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected fdatasync") - }; + fn execute_read_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Read { + sqe: sqe::Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + let mut run = |ReadSector { + page_offset, + buf_offset, + }| { + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + fd.read_page(buf, page_offset as _).map_err(Into::into) + }; + results.push(eff.traverse(run, Err)); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Read { + sqe: sqe::Read { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected read") + }; + assert!(results.len() == op_count); + let bytes_read = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + // TODO: Propagate all errors? + let result = match results.into_iter().find_map(Result::err) { + Some(error) => Err(error), + None => Ok(bytes_read), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + fn execute_open(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Open { + sqe: sqe::Open { path }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected open") + }; + let result = eff.traverse( + |()| self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }), + Err, + ); + + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_create(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Create { + sqe: sqe::Create { path }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected create") + }; + let run = |()| match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let result = eff.traverse(run, Err); + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_stat(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Stat { sqe: sqe::Stat { fd } }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected stat") + }; + let result = eff.traverse(|()| Ok(Statx { size: fd.len() }), Err); + let is_success = result.is_ok(); + self.complete(Cqe::Stat { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_fallocate(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Fallocate { + sqe: sqe::Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fallocate") + }; + let result = eff.traverse(|()| fd.set_len(total_len).map_err(Into::into), Err); + let is_success = result.is_ok(); + self.complete(Cqe::Fallocate { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_fsync(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Fsync { + sqe: sqe::Fsync { fd }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fsync") + }; + let result = eff.traverse( + |FsyncEffect::Datasync(effect)| { fd.fdatasync([effect]); - *op_count -= 1; - - *op_count == 0 - }; + Ok(()) + }, + Err, + ); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: InFlightInner::Fsync { results, .. }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fsync") + }; + // TODO: Propagate all errors? + let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let is_success = result.is_ok(); + self.complete(Cqe::Fsync { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } - if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); - self.complete(Cqe::Fdatasync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - } - Operation::Noop { sqe } => { - let InFlight { - inner: InFlightInner::Noop, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected noop") - }; - - self.complete(Cqe::Noop { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } + fn execute_fdatasync(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Fdatasync { + sqe: sqe::Fdatasync { fd }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fdatasync") + }; + let result = eff.traverse( + |effect| { + fd.fdatasync([effect]); + Ok(()) + }, + Err, + ); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: InFlightInner::Fdatasync { results, .. }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fdatasync") + }; + // TODO: Propagate all errors? + let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let is_success = result.is_ok(); + self.complete(Cqe::Fdatasync { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); } } + fn execute_noop(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected noop") + }; + let result = eff.traverse(Ok, Err); + let is_success = result.is_ok(); + self.complete(Cqe::Noop { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { if let Some(Blocked { link, diff --git a/crates/runtime-core/src/sim/io/executor/sqe.rs b/crates/runtime-core/src/sim/io/executor/sqe.rs new file mode 100644 index 00000000000..5b702b4e227 --- /dev/null +++ b/crates/runtime-core/src/sim/io/executor/sqe.rs @@ -0,0 +1,391 @@ +use alloc::{boxed::Box, vec::Vec}; + +use crate::{ + io::{ErasedBoxPtr, SECTOR_SIZE}, + sim::io::{ + executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, WriteSector}, + fs::{self, Datasync}, + Error, + }, +}; + +/// Opaque identifier of a scheduled [Sqe]. +#[derive(Clone, Copy)] +pub struct SqeId(pub(super) usize); + +impl SqeId { + pub(super) fn key(&self) -> usize { + self.0 + } +} + +/// Dependency on the previous [Sqe]. +/// +/// A link imposes an ordering constraint: the [Sqe] carrying the link will not +/// be executed before the preceding one completed. Note that a link is only +/// meaningful within a batch of SQEs submitted together. +#[derive(Clone, Copy)] +pub enum LinkKind { + /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. + /// Analogous to `IOSQE_IO_LINK`. + Soft, + /// Run the SQE regardless of the preceding SQE's result. + /// Analoguous to `IOSQE_IO_HARDLINK`. + Hard, +} + +pub struct Sqe { + pub(super) inner: SqeInner, + pub(super) link: Option, + pub(super) user_data: Option, +} + +impl Sqe { + pub fn link(mut self, kind: Option) -> Self { + self.link = kind; + self + } + + pub fn is_linked(&self) -> bool { + self.link.is_some() + } + + pub fn attach(mut self, user_data: T) -> Self { + self.user_data.replace(user_data); + self + } + + pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Write { fd, buf, offset }.into() + } + + pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Read { fd, buf, offset }.into() + } + + pub fn open(path: impl AsRef) -> Self { + Open { + path: path.as_ref().into(), + } + .into() + } + + pub fn create(path: impl AsRef) -> Self { + Create { + path: path.as_ref().into(), + } + .into() + } + + pub fn stat(fd: fs::File) -> Self { + Stat { fd }.into() + } + + pub fn fallocate(fd: fs::File, len: u64) -> Self { + Fallocate { fd, total_len: len }.into() + } + + pub fn fsync(fd: fs::File) -> Self { + Fsync { fd }.into() + } + + pub fn fdatasync(fd: fs::File) -> Self { + Fdatasync { fd }.into() + } + + pub fn noop() -> Self { + SqeInner::Noop.into() + } +} + +impl> From for Sqe { + fn from(inner: U) -> Self { + Self { + inner: inner.into(), + link: None, + user_data: None, + } + } +} + +pub enum SqeInner { + Write(Write), + Read(Read), + Open(Open), + Create(Create), + Stat(Stat), + Fallocate(Fallocate), + Fsync(Fsync), + Fdatasync(Fdatasync), + Noop, +} + +impl SqeInner { + pub(super) fn cancel(self, user_data: Option) -> Cqe { + match self { + SqeInner::Write(..) => Cqe::Write { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Read(..) => Cqe::Read { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Open(..) => Cqe::Open { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Create(..) => Cqe::Create { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Stat(..) => Cqe::Stat { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fallocate(..) => Cqe::Fallocate { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fsync(..) => Cqe::Fsync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fdatasync(..) => Cqe::Fdatasync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Noop => Cqe::Noop { + result: Err(Error::Cancelled), + user_data, + }, + } + } + + pub(super) fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { + match self { + SqeInner::Write(mut sqe) => { + let Write { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Executing { + sqe: sqe_id, + inner: Operation::WriteSector(WriteSector { + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }), + }) + .collect::>(); + let op_count = ops.len(); + let write = InFlightInner::Write { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (write, ops) + } + SqeInner::Read(mut sqe) => { + let Read { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Executing { + sqe: sqe_id, + inner: Operation::ReadSector(ReadSector { + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }), + }) + .collect::>(); + let op_count = ops.len(); + let read = InFlightInner::Read { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (read, ops) + } + SqeInner::Open(sqe) => ( + InFlightInner::Open { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Open + }], + ), + SqeInner::Create(sqe) => ( + InFlightInner::Create { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Create + }], + ), + SqeInner::Stat(sqe) => ( + InFlightInner::Stat { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Stat + }], + ), + SqeInner::Fallocate(sqe) => ( + InFlightInner::Fallocate { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Fallocate + }], + ), + SqeInner::Fsync(sqe) => { + let Fsync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Sector(offset)), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Length), + }, + }]) + .collect::>(); + let op_count = ops.len(); + let in_flight = InFlightInner::Fsync { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (in_flight, ops) + } + SqeInner::Fdatasync(sqe) => { + let Fdatasync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Sector(offset), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Length, + }, + }]) + .collect::>(); + let op_count = ops.len(); + let in_flight = InFlightInner::Fdatasync { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (in_flight, ops) + } + SqeInner::Noop => ( + InFlightInner::Noop, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Noop + }], + ), + } + } +} + +impl From for SqeInner { + fn from(inner: Write) -> Self { + Self::Write(inner) + } +} + +impl From for SqeInner { + fn from(inner: Read) -> Self { + Self::Read(inner) + } +} + +impl From for SqeInner { + fn from(inner: Open) -> Self { + Self::Open(inner) + } +} + +impl From for SqeInner { + fn from(inner: Create) -> Self { + Self::Create(inner) + } +} + +impl From for SqeInner { + fn from(inner: Stat) -> Self { + Self::Stat(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fallocate) -> Self { + Self::Fallocate(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fsync) -> Self { + Self::Fsync(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fdatasync) -> Self { + Self::Fdatasync(inner) + } +} + +pub struct Write { + pub fd: fs::File, + pub buf: ErasedBoxPtr, + pub offset: u64, +} + +pub struct Read { + pub fd: fs::File, + pub buf: ErasedBoxPtr, + pub offset: u64, +} + +pub struct Open { + pub path: Box, +} + +pub struct Create { + pub path: Box, +} + +pub struct Stat { + pub fd: fs::File, +} + +pub struct Fallocate { + pub fd: fs::File, + pub total_len: u64, +} + +pub struct Fsync { + pub fd: fs::File, +} + +pub struct Fdatasync { + pub fd: fs::File, +} diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 64aadf5a017..fd793300e34 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -105,6 +105,7 @@ impl PageMap { } } +#[derive(Clone, Copy)] pub enum Datasync { Sector(u64), Length, diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index f887ad6cf3e..4cb9003d3e1 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -10,7 +10,7 @@ use slab::Slab; use crate::{ io::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}, - sim::Rng, + sim::{io::executor::FaultInjector, Rng}, }; mod executor; @@ -60,12 +60,12 @@ pub struct SimulatorIO { } impl SimulatorIO { - pub fn tick(&self, rng: &Rng, now: Instant) -> bool { + pub fn tick(&self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let mut buffers = self.inner.buffers.lock(); - let mut progress = executor.tick(rng, now); + let mut progress = executor.tick(rng, faults); for cqe in executor.completed() { let completion = pending.remove(cqe.user_data().unwrap()); match cqe { @@ -376,7 +376,7 @@ fn reify( #[cfg(test)] mod tests { - use crate::sim::{time::TimeHandle, GlobalRng}; + use crate::sim::{io::executor::NoFaults, GlobalRng}; use super::*; @@ -384,7 +384,6 @@ mod tests { rt: tokio::runtime::LocalRuntime, io: SimulatorIO, rng: Rng, - time: TimeHandle, } impl Runtime { @@ -395,13 +394,12 @@ mod tests { .unwrap(), io: SimulatorIO::default(), rng: GlobalRng::new(0), - time: TimeHandle::default(), } } fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); - while self.io.tick(&self.rng, self.time.now()) {} + while self.io.tick(&self.rng, &mut NoFaults) {} self.rt.block_on(fut).unwrap() } } From f25388d94f608cf1afd97d214b060ab7ce690363 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 4 Sep 2026 16:55:39 +0200 Subject: [PATCH 19/19] Consider actual size written/read for Write/Read completions --- crates/runtime-core/src/io/buf.rs | 8 ++++++++ crates/runtime-core/src/sim/io/mod.rs | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-core/src/io/buf.rs index 16a8123b5c5..14b88f05c9e 100644 --- a/crates/runtime-core/src/io/buf.rs +++ b/crates/runtime-core/src/io/buf.rs @@ -115,6 +115,14 @@ mod boxed { len: self.len, } } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } } impl Drop for ErasedBox { diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 4cb9003d3e1..ce01d559fcb 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -75,7 +75,14 @@ impl SimulatorIO { }; let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(_written) => Ok(erased_buf), + Ok(written) if written == erased_buf.len() => Ok(erased_buf), + Ok(written) => Err(ErrorWith { + error: Error::ShortWrite { + expected: erased_buf.len(), + written, + }, + with: erased_buf, + }), Err(error) => Err(ErrorWith { error, with: erased_buf, @@ -89,7 +96,14 @@ impl SimulatorIO { }; let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(_written) => Ok(erased_buf), + Ok(read) if read == erased_buf.len() => Ok(erased_buf), + Ok(read) => Err(ErrorWith { + error: Error::UnexpectedEof { + expected: erased_buf.len(), + read, + }, + with: erased_buf, + }), Err(error) => Err(ErrorWith { error, with: erased_buf,