diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 29451bfc0e3c7bcba279efa7fb5fa7d40b2e6eb7..3a02a464795ec7791184a2720b877826b0d9a395 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -53,11 +53,17 @@ connection holds exclusive ownership between transactions, and a second open fai busy. No network wait occurs in a local edit. An edit a session submitted that the cache cannot write for now (a full disk, an I/O error) stays applied in memory and is written again, in order, each second; meanwhile `written` returns `WouldBlock`. A replica dropped -meanwhile keeps its cache open and its edits retrying until they are written or the cache is +meanwhile keeps its cache owned and its edits retrying until they are written or the cache is deleted, and only then releases it; `notebook::unwritten()` counts the edits waiting in the process, for a host to ask before it quits. After a commit fails, a fresh connection -decides whether it landed. An edit the cache could never write is refused alone, and the -section thread rereads the queue. +decides whether it landed. Until the cache reopens (too many open files, a busy cache), the +edits wait the same way, reads of the cache answer `WouldBlock`, and the cache stays owned: +the replica locks `-lock` and lists the cache for the process, and every open fails +busy while it does. A cache gone or no longer a database refuses every edit waiting, as an +error, and stays closed. An edit a caller blocks on, or one the cache could never write, is +refused at once, and the section thread rereads the queue; while the cache is closed, that +reread waits for it, page reads answer from the section as it was, and later edits wait to +apply to the reread section; the session then reports the section's pages as `Changed`. ## Sessions diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index 9e34335bd567e3f1569dfaa609c610405a9b8a39..c2873ef398a4aabc367daee53b91ec54579b5156 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -281,8 +281,8 @@ impl Replica { }) } - fn lock(&self) -> Result> { - lock(&self.section.connection) + fn lock(&self) -> Result> { + lock(&self.section.cache) } /// Hands `request` to the section thread. @@ -494,10 +494,41 @@ impl Drop for Replica { } } -fn lock(connection: &Mutex) -> Result> { - connection +/// The open cache's connection, for as long as its lock is held. +struct Open<'a>(MutexGuard<'a, working::Cache>); + +impl std::ops::Deref for Open<'_> { + type Target = Connection; + + fn deref(&self) -> &Connection { + match &*self.0 { + working::Cache::Open(connection) => connection, + _ => unreachable!("Only an open cache is lent"), + } + } +} + +impl std::ops::DerefMut for Open<'_> { + fn deref_mut(&mut self) -> &mut Connection { + match &mut *self.0 { + working::Cache::Open(connection) => connection, + _ => unreachable!("Only an open cache is lent"), + } + } +} + +/// The cache's connection; `WouldBlock` while a failed commit keeps the cache closed. +fn lock(cache: &Mutex) -> Result> { + let cache = cache .lock() - .map_err(|_| io::Error::other("Cache owner panicked").into()) + .map_err(|_| io::Error::other("Cache owner panicked"))?; + match &*cache { + working::Cache::Open(_) => Ok(Open(cache)), + working::Cache::Closed { .. } => { + Err(io::Error::new(io::ErrorKind::WouldBlock, "The cache reopens once it can").into()) + } + working::Cache::Lost(error) => Err(io::Error::other(error.clone()).into()), + } } /// Tells the worker a burst of local edits is durable. @@ -558,9 +589,101 @@ fn signed(value: u64) -> Result { Ok(i64::try_from(value).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?) } +/// `connect`, busy while anyone owns the cache: SQLite's lock, or the `Owned` a section holds +/// while a failed commit keeps the cache closed, checked once SQLite's is held, as that +/// section takes its own before letting SQLite's go. +fn cache_connection(path: &Path) -> Result { + let connection = connect(path)?; + connection.query_row("SELECT count(*) FROM sqlite_schema", [], |_| Ok(()))?; + let owned = identity(path) + .is_ok_and(|identity| OWNED.lock().is_ok_and(|owned| owned.contains(&identity))) + || std::fs::File::open(owner_lock(path)).is_ok_and(|lock| { + matches!( + lock.try_lock_shared(), + Err(std::fs::TryLockError::WouldBlock) + ) + }); + if owned { + return Err(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_BUSY), + Some("A section holds the cache while it reopens".into()), + ) + .into()); + } + Ok(connection) +} + +/// The file beside a cache whose lock keeps it owned while a failed commit keeps it closed. +fn owner_lock(cache: &Path) -> PathBuf { + let mut lock = cache.as_os_str().to_owned(); + lock.push("-lock"); + lock.into() +} + +/// Caches an `Owned` holds in this process, where a lock on a network filesystem (`flock` over +/// NFS on Linux) need not exclude. +static OWNED: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + +/// A cache file however its path is spelled: its device and inode, or elsewhere its +/// canonical path. +#[cfg(unix)] +type Identity = (u64, u64); +#[cfg(not(unix))] +type Identity = PathBuf; + +fn identity(cache: &Path) -> io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(cache)?; + Ok((metadata.dev(), metadata.ino())) + } + #[cfg(not(unix))] + std::fs::canonicalize(cache) +} + +/// Ownership of a closed cache, which every open honours until it drops: `-lock` +/// locked, and the cache listed in `OWNED`. +struct Owned { + cache: PathBuf, + identity: Identity, + _lock: std::fs::File, +} + +impl Owned { + fn take(cache: &Path) -> Result { + let lock = std::fs::File::options() + .create(true) + .truncate(false) + .write(true) + .open(owner_lock(cache))?; + lock.try_lock().map_err(io::Error::from)?; + let identity = identity(cache)?; + #[cfg_attr(unix, allow(clippy::clone_on_copy))] + (OWNED.lock()) + .map_err(|_| io::Error::other("Cache owners panicked"))? + .insert(identity.clone()); + Ok(Self { + cache: cache.to_owned(), + identity, + _lock: lock, + }) + } +} + +impl Drop for Owned { + fn drop(&mut self) { + if let Ok(mut owned) = OWNED.lock() { + owned.remove(&self.identity); + } + let _ = std::fs::remove_file(owner_lock(&self.cache)); + } +} + /// Opens a cache without writing to it: exclusive locking, a full sync of every commit /// (`F_FULLFSYNC` on macOS, the WAL's syncs included) and foreign keys, each queried back. -fn cache_connection(path: &Path) -> Result { +fn connect(path: &Path) -> Result { #[cfg(target_arch = "wasm32")] fs::install_sqlite(); let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; diff --git a/crates/notebook/src/location.rs b/crates/notebook/src/location.rs index 6a138373dd3a463078bf509e338448c0ea59337a..f72a9adb1c1f92b5b73e63bce8f603c08ab21d56 100644 --- a/crates/notebook/src/location.rs +++ b/crates/notebook/src/location.rs @@ -191,13 +191,23 @@ fn adopt(from: &Path, to: &Path) -> io::Result<()> { Ok(()) } -/// Removes a replica folder that holds no replica. +/// Removes a replica folder that holds no replica, with the owner lock a crash may have left +/// beside a cache moved away since (`crate::Owned`). fn tidy(folder: &Path) { - let empty = fs::read_dir(folder).is_ok_and(|mut entries| { - entries.all(|entry| entry.is_ok_and(|entry| entry.file_name() == LOCATION)) - }); - if empty { - let _ = fs::remove_file(folder.join(LOCATION)); + let Ok(entries) = fs::read_dir(folder).and_then(|entries| { + entries + .map(|entry| Ok(entry?.file_name())) + .collect::>>() + }) else { + return; + }; + let left = |name: &std::ffi::OsString| { + name == LOCATION || name.to_string_lossy().ends_with(".sqlite-lock") + }; + if entries.iter().all(left) { + for name in entries { + let _ = fs::remove_file(folder.join(name)); + } let _ = fs::remove_dir(folder); } } diff --git a/crates/notebook/src/session.rs b/crates/notebook/src/session.rs index 9b4880989ff1fa5044cc2adbca50c84819b0f769..4cbf57ba5810b73cc9c7edc126ef8d5308bad2e6 100644 --- a/crates/notebook/src/session.rs +++ b/crates/notebook/src/session.rs @@ -2186,7 +2186,8 @@ pub struct SyncStatus { pub synced: Option, /// Why the last step failed, until one reaches the section file again. pub error: Option, - /// Edits the section file does not hold yet, uncertain attempts included. + /// Edits the section file does not hold yet, uncertain attempts and edits waiting for the + /// cache included; while a failed commit keeps the cache closed, those queued before it. pub queued: u64, } @@ -2237,11 +2238,23 @@ pub(crate) fn sync_status( synced: impl FnOnce() -> Option, observed: &Mutex, ) -> Result { - let queued: i64 = crate::lock(&replica.section.connection)?.query_row( - "SELECT count(*) FROM edits", - [], - |row| row.get(0), - )?; + let waiting = replica + .section + .waiting + .load(std::sync::atomic::Ordering::Relaxed) as u64; + let (queued, failed) = match &*(replica.section.cache.lock()) + .map_err(|_| io::Error::other("Cache owner panicked"))? + { + crate::working::Cache::Open(connection) => { + let queued: i64 = + connection.query_row("SELECT count(*) FROM edits", [], |row| row.get(0))?; + (crate::unsigned(queued)?, None) + } + crate::working::Cache::Closed { queued, .. } => { + (*queued, Some("The cache reopens once it can".to_owned())) + } + crate::working::Cache::Lost(error) => (0, Some(error.clone())), + }; let observed = observed .lock() .map_err(|_| io::Error::other("Synchronization observer panicked"))?; @@ -2249,11 +2262,11 @@ pub(crate) fn sync_status( // Read after `observed`: the worker stores the time before it clears the error, so a // cleared error never shows beside the time of the step before it. synced: synced(), - error: observed - .as_ref() - .and_then(Option::as_ref) - .map(|error| io::Error::new(error.kind(), error.to_string())), - queued: crate::unsigned(queued)?, + error: failed.map(io::Error::other).or_else(|| { + (observed.as_ref().and_then(Option::as_ref)) + .map(|error| io::Error::new(error.kind(), error.to_string())) + }), + queued: queued + waiting, }) } @@ -2475,6 +2488,14 @@ impl Section { } }); let observed: Arc> = Arc::new(Mutex::new(None)); + if let Ok(mut reread) = replica.section.reread.lock() { + let (sender, notify) = (sender.clone(), Arc::clone(¬ify)); + *reread = Some(Box::new(move |spaces| { + if sender.send(Event::Changed(spaces)).is_ok() { + notify(); + } + })); + } let mut section = Self { file, replica, diff --git a/crates/notebook/src/session/tests.rs b/crates/notebook/src/session/tests.rs index db6d36940f8a936e4f307cff89c3644ac2278604..e922fb84a50214a546c27ed10dfe8f16d47df4a7 100644 --- a/crates/notebook/src/session/tests.rs +++ b/crates/notebook/src/session/tests.rs @@ -854,3 +854,24 @@ fn edits_the_cache_refuses_wait_in_order_and_land_once() { assert_eq!(waited[0].page.color, Some(0x0065_4321)); assert!(Replica::open(&cache).unwrap().pending().unwrap().is_empty()); } + +#[test] +fn a_cache_owned_while_closed_opens_for_nobody_else() { + let temporary = tempfile::tempdir().unwrap(); + let cache = temporary.path().join("owned.sqlite"); + let source = onestore::create_section("Owned.one", "Original", "Fixture").unwrap(); + drop(Replica::create(&cache, &source).unwrap()); + let named = PathBuf::from(crate::connect(&cache).unwrap().path().unwrap()); + let owned = crate::Owned::take(&named).unwrap(); + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + assert!(crate::closed(&cache).is_err_and(|error| error.busy())); + drop(owned); + assert!(!crate::owner_lock(&named).exists()); + drop(Replica::open(&cache).unwrap()); + // Another process holds only the lock beside the cache. + let lock = std::fs::File::create(crate::owner_lock(&named)).unwrap(); + lock.try_lock().unwrap(); + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + drop(lock); + drop(Replica::open(&cache).unwrap()); +} diff --git a/crates/notebook/src/working.rs b/crates/notebook/src/working.rs index 99fa5e52ea9e671447ee0a7d0d209834665fe3db..21e898b8f3cb374672f0776be59220989870ef50 100644 --- a/crates/notebook/src/working.rs +++ b/crates/notebook/src/working.rs @@ -15,7 +15,7 @@ use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, io, - path::PathBuf, + path::{Path, PathBuf}, sync::{ Arc, Mutex, Weak, atomic::{AtomicUsize, Ordering}, @@ -25,6 +25,7 @@ use std::{ pub(crate) type Reply = Box) + Send>; pub(crate) type Notice = Box; +type Changed = Box) + Send>; pub(crate) enum Request { Apply(Accepted), @@ -87,6 +88,34 @@ pub(crate) struct Sealed { pub transaction: Option, } +/// The cache's connection, or what a failed commit leaves: the cache closed until it reopens +/// and reads cold whether the commit whose last edit is `last` landed, `ids` naming its edits +/// still waiting and `queued` the edits queued before it; or lost where it never will reopen. +pub(crate) enum Cache { + Open(Connection), + Closed { + last: u64, + ids: Vec, + queued: u64, + owner: Owner, + }, + Lost(String), +} + +/// What keeps everyone else out of a closed cache: the warm connection, unused, until +/// `crate::Owned` can be taken in its stead. +pub(crate) enum Owner { + Warm(#[allow(dead_code)] Connection), + Owned(#[allow(dead_code)] crate::Owned), +} + +/// Whether a failed commit keeps the cache closed. +fn closed(cache: &Mutex) -> bool { + cache + .lock() + .is_ok_and(|cache| matches!(*cache, Cache::Closed { .. })) +} + impl Request { fn fail(self, kind: io::ErrorKind, message: &str) { let error = || io::Error::new(kind, message.to_owned()).into(); @@ -112,7 +141,10 @@ impl Request { /// keeps answering page reads from the section as it was; the new thread then takes the /// requests over, so reads never wait for a rebuild. pub(crate) struct Thread { - pub(crate) connection: Mutex, + pub(crate) cache: Mutex, + /// Where the cache reopens after a failed commit; none in the browser, where reopening + /// through its file system is unverified. + path: Option, /// The sync worker, which a durable burst of edits wakes. pub(crate) worker: Mutex>, /// Taken when the replica drops, which ends every section thread. @@ -126,7 +158,9 @@ pub(crate) struct Thread { /// A password-protected section's key, under which its queue is sealed too. pub(crate) key: Option, /// The edits the cache last refused for now, as `crate::unwritten` counts them. - waiting: AtomicUsize, + pub(crate) waiting: AtomicUsize, + /// Told the pages a reread after a refusal may change, which their views reload. + pub(crate) reread: Mutex>, /// Last, so the cache closes before background synchronization takes it over. pub(crate) released: crate::Released, } @@ -309,18 +343,20 @@ mod serving { } } -/// The next request; a second passes without one while `retry`. +/// How often the cache is tried again while edits wait for it or it is closed. +const RETRY: std::time::Duration = std::time::Duration::from_secs(1); + +/// The next request, or a timeout once `retry` passes without one. #[cfg(not(target_arch = "wasm32"))] async fn receive( requests: &mpsc::Receiver, - retry: bool, + retry: Option, ) -> std::result::Result { - if retry { - requests.recv_timeout(std::time::Duration::from_secs(1)) - } else { - requests + match retry { + Some(wait) => requests.recv_timeout(wait), + None => requests .recv() - .map_err(|_| mpsc::RecvTimeoutError::Disconnected) + .map_err(|_| mpsc::RecvTimeoutError::Disconnected), } } @@ -329,7 +365,7 @@ async fn receive( #[cfg(target_arch = "wasm32")] async fn receive( requests: &mpsc::Receiver, - _retry: bool, + _retry: Option, ) -> std::result::Result { std::future::poll_fn(|_| match requests.try_recv() { Ok(request) => std::task::Poll::Ready(Ok(request)), @@ -349,7 +385,10 @@ pub(crate) fn spawn(connection: Connection, key: Option) -> Result<(Arc) -> Result<(Arc working, Err(error) => { if let Some(ready) = ready.take() { @@ -451,19 +491,17 @@ enum Job { fn build(shared: Arc, job: Job, signal: mpsc::Sender) { let answer: Box = match job { Job::Reopen { reply } => Box::new(move || reply(Ok(()))), - Job::Rebase { image, reply } => { - match rebase(&shared.connection, image, shared.key.as_ref()) { - Ok(changed) => Box::new(move || reply(Ok(changed))), - Err(error) => { - reply(Err(error)); - let _ = signal.send(Request::Resume); - return; - } + Job::Rebase { image, reply } => match rebase(&shared.cache, image, shared.key.as_ref()) { + Ok(changed) => Box::new(move || reply(Ok(changed))), + Err(error) => { + reply(Err(error)); + let _ = signal.send(Request::Resume); + return; } - } + }, }; let arena = Arena::default(); - let opened = Working::open(&arena, &shared.connection, shared.key.as_ref()); + let opened = Working::open(&arena, &shared.cache, shared.key.as_ref()); let (to, from) = mpsc::sync_channel(1); if signal.send(Request::Handover(to)).is_err() { return; @@ -492,40 +530,65 @@ fn build(shared: Arc, job: Job, signal: mpsc::Sender) { fn inline(shared: &Thread, job: Job) -> bool { match job { Job::Reopen { reply } => reply(Ok(())), - Job::Rebase { image, reply } => { - match rebase(&shared.connection, image, shared.key.as_ref()) { - Ok(changed) => reply(Ok(changed)), - Err(error) => { - reply(Err(error)); - return false; - } + Job::Rebase { image, reply } => match rebase(&shared.cache, image, shared.key.as_ref()) { + Ok(changed) => reply(Ok(changed)), + Err(error) => { + reply(Err(error)); + return false; } - } + }, } true } -/// Writes again each second the edits a dropped replica left waiting, until none do or their -/// cache is gone, keeping the cache open and `Released` from firing meanwhile. +/// Writes again each second the edits a dropped replica left waiting, until none do and the +/// cache is open, or the cache is gone; the cache stays owned and `Released` unfired meanwhile. +/// With `reread`, the closed commit is settled first and the edits applied again to the queue +/// as it then stands, refusing those that no longer apply, as the section thread would have. #[cfg(not(target_arch = "wasm32"))] -fn orphan(shared: &Arc, unwritten: Vec) { +fn orphan(shared: &Arc, mut open: Option, mut unwritten: Vec, reread: bool) { let shared = Arc::clone(shared); - let path = lock(&shared.connection) - .ok() - .and_then(|connection| connection.path().map(PathBuf::from)); let spawned = std::thread::Builder::new() .name("onestore-unwritten".into()) .spawn(move || { - let arena = Arena::default(); - let Ok(mut working) = Working::open(&arena, &shared.connection, shared.key.as_ref()) - else { - return; - }; - working.unwritten = unwritten; // A cache deleted, as with its notebook, takes no edits. - while !working.unwritten.is_empty() && path.as_ref().is_some_and(|path| path.exists()) { - std::thread::sleep(std::time::Duration::from_secs(1)); - let _ = working.flush(&shared); + let kept = || shared.path.as_ref().is_some_and(|path| path.exists()); + let second = || std::thread::sleep(std::time::Duration::from_secs(1)); + while closed(&shared.cache) && kept() { + second(); + let _ = write(&shared, &mut open, &mut unwritten, !reread); + } + // An edit that applies part way is refused, and the others apply again to a fresh + // reading of the queue. + if reread { + 'reread: loop { + let arena = Arena::default(); + let mut working = + match Working::open(&arena, &shared.cache, shared.key.as_ref()) { + Ok(working) => working, + Err(error) => { + for edit in unwritten.drain(..) { + (edit.reply)(Err(io::Error::other(error.to_string()).into())); + } + break; + } + }; + let mut edits = std::mem::take(&mut unwritten).into_iter(); + while let Some(edit) = edits.next() { + if working.accept(edit) { + unwritten = std::mem::take(&mut working.unwritten); + unwritten.extend(edits); + continue 'reread; + } + } + open = working.open; + unwritten = std::mem::take(&mut working.unwritten); + break; + } + } + while !unwritten.is_empty() && kept() { + second(); + let _ = write(&shared, &mut open, &mut unwritten, true); } }); if let Err(error) = spawned { @@ -556,7 +619,7 @@ struct Working<'a> { } impl<'a> Working<'a> { - fn open(arena: &'a Arena, connection: &Mutex, key: Option<&Key>) -> Result { + fn open(arena: &'a Arena, connection: &Mutex, key: Option<&Key>) -> Result { let (section, open, touched) = replay(arena, &*lock(connection)?, key)?; Ok(Self { section, @@ -572,23 +635,53 @@ impl<'a> Working<'a> { requests: &mpsc::Receiver, backlog: &mut VecDeque, ) -> Next { - let connection = &shared.connection; + let connection = &shared.cache; // While another thread rebuilds the section, reads answer from this one and every // other request waits for the rebuilt section. let mut held: Option> = None; + // While a reread waits for the closed cache, reads answer from this section and edits + // wait for the reread, as one this section refused may underlie them. + let mut stale: Option> = None; + let mut attempted = web_time::Instant::now(); loop { let mut burst = std::mem::take(backlog); - // Only a burst that took an edit, or a retry's tick, writes at its end: reads never. + // Only a burst that took an edit, or a retry falling due, writes at its end: reads + // never. let mut write = false; + let retry = !self.unwritten.is_empty() || closed(connection); if burst.is_empty() { - match receive(requests, !self.unwritten.is_empty()).await { + let wait = retry.then(|| RETRY.saturating_sub(attempted.elapsed())); + match receive(requests, wait).await { Ok(request) => burst.push_back(request), - Err(mpsc::RecvTimeoutError::Timeout) => write = true, + Err(mpsc::RecvTimeoutError::Timeout) => {} Err(mpsc::RecvTimeoutError::Disconnected) => { - let _ = self.flush(shared); + let stale_section = stale.is_some(); + // A refusal now leaves the section holding an edit the queue lacks too. + #[cfg_attr(target_arch = "wasm32", allow(unused_variables))] + let reread = crate::working::write( + shared, + &mut self.open, + &mut self.unwritten, + !stale_section, + ) + .is_err() + || stale_section; + for request in stale.into_iter().flatten() { + match request { + Request::Apply(accepted) => self.unwritten.push(accepted), + request => { + request.fail(io::ErrorKind::Other, "The section thread stopped") + } + } + } #[cfg(not(target_arch = "wasm32"))] - if !self.unwritten.is_empty() { - orphan(shared, std::mem::take(&mut self.unwritten)); + if !self.unwritten.is_empty() || closed(connection) { + orphan( + shared, + self.open, + std::mem::take(&mut self.unwritten), + reread, + ); } return Next::Stop; } @@ -599,24 +692,11 @@ impl<'a> Working<'a> { let mut later = VecDeque::new(); let mut reopen = false; while !reopen && let Some(request) = burst.pop_front().or_else(|| later.pop_front()) { + let Some(request) = self.answer(request) else { + continue; + }; if let Some(waiting) = &mut held { match request { - Request::Page { space, reply } => { - reply(self.section.page(space).map_err(Into::into)) - } - Request::Pages { reply } => reply(self.section.pages().map_err(Into::into)), - Request::Conflicts { reply } => { - reply(self.section.conflicts().map_err(Into::into)) - } - Request::Versions { reply } => { - reply(self.section.versions().map_err(Into::into)) - } - Request::Version { - space, - version, - reply, - } => reply(self.section.version(space, version).map_err(Into::into)), - Request::Moving { space, reply } => reply(self.moving(space)), Request::Handover(to) => { let mut waiting = held.take().unwrap_or_default(); waiting.extend(burst.drain(..).chain(later.drain(..))); @@ -630,6 +710,25 @@ impl<'a> Working<'a> { } continue; } + if let Some(waiting) = &mut stale { + match request { + Request::Apply( + accepted @ Accepted { + waiting: Some(_), .. + }, + ) => { + if let Some(notice) = &accepted.waiting { + notice("The cache reopens once it can"); + } + waiting.push_back(Request::Apply(accepted)); + } + request @ Request::Reopen { .. } => waiting.push_back(request), + request => { + request.fail(io::ErrorKind::WouldBlock, "The cache reopens once it can") + } + } + continue; + } if matches!(request, Request::Flush { .. } | Request::Seal { .. }) && !burst.is_empty() { @@ -641,7 +740,7 @@ impl<'a> Working<'a> { Request::Flush { .. } | Request::Seal { .. } | Request::Rebase { .. } ) { let failed = self.flush(shared).is_err(); - if failed || !self.unwritten.is_empty() { + if failed || !self.unwritten.is_empty() || closed(connection) { // The worker retries later; the edits wait here meanwhile. request.fail(io::ErrorKind::WouldBlock, "Edits wait to be written"); reopen = failed; @@ -650,36 +749,17 @@ impl<'a> Working<'a> { } reopen = match request { Request::Apply(accepted) => { - write = true; - self.accept(accepted) - } - Request::Page { space, reply } => { - reply(self.section.page(space).map_err(Into::into)); - false - } - Request::Pages { reply } => { - reply(self.section.pages().map_err(Into::into)); - false - } - Request::Conflicts { reply } => { - reply(self.section.conflicts().map_err(Into::into)); - false - } - Request::Versions { reply } => { - reply(self.section.versions().map_err(Into::into)); - false - } - Request::Version { - space, - version, - reply, - } => { - reply(self.section.version(space, version).map_err(Into::into)); - false - } - Request::Moving { space, reply } => { - reply(self.moving(space)); - false + // While the cache is closed, an edit nobody waits for is refused before + // it applies, instead of showing until a reread can drop it. + if accepted.waiting.is_none() + && let Err(error) = lock(connection).map(drop) + { + (accepted.reply)(Err(error)); + false + } else { + write = true; + self.accept(accepted) + } } Request::Flush { reply, durable } => { #[cfg(target_arch = "wasm32")] @@ -715,6 +795,12 @@ impl<'a> Working<'a> { } Request::Reopen { reply } => { let _ = self.flush(shared); + if closed(connection) { + stale + .get_or_insert_default() + .push_back(Request::Reopen { reply }); + continue; + } #[cfg(target_arch = "wasm32")] let reopen = inline(shared, Job::Reopen { reply }); #[cfg(not(target_arch = "wasm32"))] @@ -726,20 +812,86 @@ impl<'a> Working<'a> { }; reopen } - Request::Handover(_) | Request::Resume => false, + Request::Page { .. } + | Request::Pages { .. } + | Request::Conflicts { .. } + | Request::Versions { .. } + | Request::Version { .. } + | Request::Moving { .. } + | Request::Handover(_) + | Request::Resume => false, }; } - if reopen || write && self.flush(shared).is_err() { - backlog.extend( - self.unwritten - .drain(..) - .map(Request::Apply) - .chain(burst.drain(..)) - .chain(later.drain(..)), - ); - return Next::Reopen; + // Retries fall due once a second however busy the section is; a closed cache is + // tried then alone, not with each edit. + if retry { + let due = attempted.elapsed() >= RETRY; + write = if closed(connection) { + due + } else { + write || due + }; + } + if write { + attempted = web_time::Instant::now(); + } + if let Some(waiting) = &stale { + // Only whether the closed commit landed is settled before the reread. + if write { + let _ = + crate::working::write(shared, &mut self.open, &mut self.unwritten, false); + } + if closed(connection) { + let held = waiting + .iter() + .filter(|request| matches!(request, Request::Apply(_))) + .count(); + shared.wait(self.unwritten.len() + held); + continue; + } + } else if reopen || write && self.flush(shared).is_err() { + if closed(connection) { + stale = Some(VecDeque::new()); + backlog.extend(burst.drain(..).chain(later.drain(..))); + continue; + } + } else { + continue; } + backlog.extend( + self.unwritten + .drain(..) + .map(Request::Apply) + .chain(stale.into_iter().flatten()) + .chain(burst.drain(..)) + .chain(later.drain(..)), + ); + // What the host asks next answers from the reread section. + if let (Ok(pages), Ok(reread)) = (self.section.pages(), shared.reread.lock()) + && let Some(reread) = reread.as_ref() + { + reread(pages.into_iter().map(|(space, ..)| space).collect()); + } + return Next::Reopen; + } + } + + /// Answers `request` where it reads the section; any other comes back. + fn answer(&mut self, request: Request) -> Option { + match request { + Request::Page { space, reply } => reply(self.section.page(space).map_err(Into::into)), + Request::Pages { reply } => reply(self.section.pages().map_err(Into::into)), + Request::Conflicts { reply } => reply(self.section.conflicts().map_err(Into::into)), + Request::Versions { reply } => reply(self.section.versions().map_err(Into::into)), + Request::Version { + space, + version, + reply, + } => reply(self.section.version(space, version).map_err(Into::into)), + Request::Moving { space, reply } => reply(self.moving(space)), + request => return Some(request), } + None } /// Starts a thread that runs `job` and rereads the section; false when the replica is @@ -807,109 +959,12 @@ impl<'a> Working<'a> { } } - /// Writes the unwritten edits in one transaction, then answers their senders. A write - /// that may succeed later keeps the edits with a notice, telling the first; one that never - /// will refuses the edit it failed on. An error means the section holds an edit the queue - /// never will, or the cache closed, and must be reread. fn flush(&mut self, shared: &Arc) -> Result<()> { - if self.unwritten.is_empty() { - return Ok(()); - } - let mut connection = lock(&shared.connection)?; - let mut ids = Vec::new(); - let committed = connection - .transaction_with_behavior(TransactionBehavior::Immediate) - .map_err(crate::Error::from) - .and_then(|transaction| { - let batch = match self.open { - Some(batch) => batch, - None => { - transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; - transaction.last_insert_rowid() - } - }; - for edit in &self.unwritten { - ids.push(queue::insert( - &transaction, - shared.key.as_ref(), - None, - batch, - &edit.author, - &edit.edit, - )?); - } - Ok((batch, transaction.commit().err())) - }); - // The error, and the edit it refuses when it would fail again. - let written = match committed { - Ok((batch, None)) => Ok(batch), - Ok((batch, Some(error))) => { - if landed(&mut connection, ids.last().copied().unwrap_or_default())? { - Ok(batch) - } else { - Err((error.into(), None)) - } - } - Err(error) => { - let refused = (!transient(&error)).then_some(ids.len()); - Err((error, refused)) - } - }; - drop(connection); - let batch = match written { - Ok(batch) => batch, - Err((error, refused)) => { - let message = error.to_string(); - let before = self.unwritten.len(); - let mut at = 0; - for edit in self.unwritten.extract_if(.., |edit| { - at += 1; - edit.waiting.is_none() || refused == Some(at - 1) - }) { - (edit.reply)(Err(io::Error::other(message.clone()).into())); - } - shared.wait(self.unwritten.len()); - if let Some(notice) = self - .unwritten - .first() - .and_then(|edit| edit.waiting.as_ref()) - { - notice(&message); - } - return if self.unwritten.len() == before { - Ok(()) - } else { - Err(error) - }; - } - }; - self.open = Some(batch); - #[cfg(target_arch = "wasm32")] - shared.receipt( - self.unwritten - .drain(..) - .zip(ids) - .map(|(edit, id)| { - let reply: Reply<()> = - Box::new(move |result| (edit.reply)(result.map(|()| id))); - reply - }) - .collect(), - true, - ); - #[cfg(not(target_arch = "wasm32"))] - { - for (edit, id) in self.unwritten.drain(..).zip(ids) { - (edit.reply)(Ok(id)); - } - crate::edited(&shared.worker); - } - shared.wait(0); - Ok(()) + write(shared, &mut self.open, &mut self.unwritten, true) } /// Seals the open batch unless a sealed batch still waits for publication. - fn seal(&mut self, connection: &Mutex) -> Result> { + fn seal(&mut self, connection: &Mutex) -> Result> { let Some(batch) = self.open else { return Ok(None); }; @@ -954,7 +1009,7 @@ impl<'a> Working<'a> { /// version the remote could not take gains a conflict page holding it, queued as a new edit. /// Returns the pages the remote changed. fn rebase( - connection: &Mutex, + connection: &Mutex, image: Option>, key: Option<&Key>, ) -> Result> { @@ -1186,28 +1241,206 @@ fn transient(error: &crate::Error) -> bool { if matches!(error.code, SystemIoFailure | DiskFull | DatabaseBusy | DatabaseLocked | CannotOpen)) } -/// Whether the commit whose last edit is `last` reached the cache after it failed, as a cold -/// open reads it, a commit landing whole: the warm connection may not know of frames its WAL -/// holds, and must never write again. When the cache cannot be reopened, an inert connection -/// stays, failing every statement. -fn landed(connection: &mut Connection, last: u64) -> Result { - let path = connection.path().map(PathBuf::from); - let inert = Connection::open_in_memory()?; - inert.execute_batch("PRAGMA query_only=1")?; - std::mem::replace(connection, inert) - .close() - .map_err(|(_, error)| error)?; +/// Writes `unwritten` in one transaction after the open batch `open`, then answers their +/// senders. A write that may succeed later keeps the edits with a notice; one that never will +/// refuses the edit it failed on, and an edit nobody waits for is refused at once. A failed +/// commit leaves the cache closed, still owned, until it reopens to decide whether the commit +/// landed; a lost cache refuses every edit. An error means the section holds an edit the +/// queue never will, or the cache is lost, and must be reread. +fn write( + shared: &Arc, + open: &mut Option, + unwritten: &mut Vec, + commit: bool, +) -> Result<()> { + let mut cache = (shared.cache.lock()).map_err(|_| io::Error::other("Cache owner panicked"))?; + if unwritten.is_empty() && !matches!(*cache, Cache::Closed { .. }) { + return Ok(()); + } + let path = shared.path.as_deref(); + // The ids the cache holds for the first of the edits. + let mut ids = Vec::new(); + // The error, and the edit it refuses when it would fail again. + let failed = (|| { + ids = reopen(&mut cache, path, open).map_err(|error| (error, None))?; + if !commit || ids.len() == unwritten.len() { + return Ok(()); + } + let Cache::Open(connection) = &mut *cache else { + unreachable!("A cache that does not reopen fails"); + }; + let mut inserted = Vec::new(); + let mut queued = 0; + let committed = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(crate::Error::from) + .and_then(|transaction| { + queued = transaction + .query_row("SELECT count(*) FROM edits", [], |row| row.get::<_, i64>(0))?; + let batch = match *open { + Some(batch) => batch, + None => { + transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; + transaction.last_insert_rowid() + } + }; + for edit in &unwritten[ids.len()..] { + inserted.push(queue::insert( + &transaction, + shared.key.as_ref(), + None, + batch, + &edit.author, + &edit.edit, + )?); + } + Ok((batch, transaction.commit().err())) + }); + match committed { + Ok((batch, None)) => *open = Some(batch), + Ok((_, Some(error))) => { + // The warm connection may not know of frames its WAL holds, and must never + // write again; it keeps the cache owned until the owner lock can. + if let Cache::Open(warm) = + std::mem::replace(&mut *cache, Cache::Lost(String::new())) + { + *cache = Cache::Closed { + last: inserted[inserted.len() - 1], + ids: inserted, + queued: queued.unsigned_abs(), + owner: Owner::Warm(warm), + }; + } + inserted = reopen(&mut cache, path, open).map_err(|error| (error, None))?; + if inserted.is_empty() { + return Err((error.into(), None)); + } + } + Err(error) => { + let refused = (!transient(&error)).then_some(inserted.len()); + return Err((error, refused)); + } + } + ids.extend(inserted); + Ok(()) + })(); + let written: Vec<_> = unwritten.drain(..ids.len()).zip(ids).collect(); + let refused: Vec<_> = match &failed { + Ok(()) => Vec::new(), + Err((_, failing)) => { + let lost = matches!(*cache, Cache::Lost(_)); + let (mut at, mut kept) = (0, 0); + (unwritten.extract_if(.., |edit| { + at += 1; + let refuse = lost || edit.waiting.is_none() || *failing == Some(at - 1); + if !refuse { + kept += 1; + } else if let Cache::Closed { ids, .. } = &mut *cache + && kept < ids.len() + { + // Whether its commit landed, it is answered now; a reread shows which. + ids.remove(kept); + } + refuse + })) + .collect() + } + }; + drop(cache); + // Counted before anyone hears, as a host hearing may ask. + shared.wait(unwritten.len()); + if !written.is_empty() { + #[cfg(target_arch = "wasm32")] + shared.receipt( + written + .into_iter() + .map(|(edit, id)| { + let reply: Reply<()> = + Box::new(move |result| (edit.reply)(result.map(|()| id))); + reply + }) + .collect(), + true, + ); + #[cfg(not(target_arch = "wasm32"))] + { + for (edit, id) in written { + (edit.reply)(Ok(id)); + } + crate::edited(&shared.worker); + } + } + let Err((error, _)) = failed else { + return Ok(()); + }; + let message = error.to_string(); + let kept = refused.is_empty(); + for edit in refused { + (edit.reply)(Err(io::Error::other(message.clone()).into())); + } + if let Some(notice) = unwritten.iter().find_map(|edit| edit.waiting.as_ref()) { + notice(&message); + } + if kept { Ok(()) } else { Err(error) } +} + +/// Reopens the cache a failed commit closed and reads cold whether the commit landed, as one +/// lands whole: the ids of its edits still waiting when it did, its batch becoming `open`; +/// none where it did not or the cache was open. The closed cache stays owned meanwhile, and +/// is lost once the file is gone or no longer a database. +fn reopen(cache: &mut Cache, path: Option<&Path>, open: &mut Option) -> Result> { + let (last, ids, owner) = match cache { + Cache::Open(_) => return Ok(Vec::new()), + Cache::Closed { + last, ids, owner, .. + } => (*last, ids, owner), + Cache::Lost(error) => return Err(io::Error::other(error.clone()).into()), + }; // Reopening through the browser's file system is unverified: it stays closed. - let path = path - .filter(|_| cfg!(not(target_arch = "wasm32"))) - .ok_or_else(|| io::Error::other("The cache could not confirm a write"))?; - let cold = crate::cache_connection(&path)?; - crate::write_ahead(&cold)?; - let found: bool = cold.query_row( - "SELECT EXISTS(SELECT 1 FROM edits WHERE id=?1)", - [crate::signed(last)?], - |row| row.get(0), - )?; - *connection = cold; - Ok(found) + let Some(path) = path else { + let error = "The cache could not confirm a write"; + *cache = Cache::Lost(error.to_owned()); + return Err(io::Error::other(error).into()); + }; + let reopened = (|| -> Result<(Connection, Option)> { + if let Owner::Warm(_) = owner { + *owner = Owner::Owned(crate::Owned::take(path)?); + } + let cold = crate::connect(path)?; + crate::write_ahead(&cold)?; + if cold.is_readonly(rusqlite::MAIN_DB)? { + return Err(io::Error::from(io::ErrorKind::ReadOnlyFilesystem).into()); + } + let batch = cold + .query_row( + "SELECT batch FROM edits WHERE id=?1", + [crate::signed(last)?], + |row| row.get(0), + ) + .optional()?; + Ok((cold, batch)) + })(); + match reopened { + Ok((cold, batch)) => { + let landed = match batch { + Some(batch) => { + *open = Some(batch); + std::mem::take(ids) + } + None => Vec::new(), + }; + *cache = Cache::Open(cold); + Ok(landed) + } + Err(error) => { + use rusqlite::ErrorCode::*; + let lost = !path.exists() + || matches!(&error, crate::Error::Database(rusqlite::Error::SqliteFailure(error, _)) + if matches!(error.code, NotADatabase | DatabaseCorrupt)); + if lost { + *cache = Cache::Lost(error.to_string()); + } + Err(error) + } + } } diff --git a/crates/notebook/tests/commit_fault.rs b/crates/notebook/tests/commit_fault.rs index a6fd97be281971bd15b274183332f4a8a1b62b50..cd5592f81b02144ff26f445882de969e2da9ec86 100644 --- a/crates/notebook/tests/commit_fault.rs +++ b/crates/notebook/tests/commit_fault.rs @@ -1,20 +1,47 @@ -//! Commits whose WAL write or sync fails, through a shim over the platform's VFS. Its own -//! binary, as the shim becomes the process's default VFS. +//! Commits whose WAL write or sync fails, and caches that will not reopen after one, through a +//! shim over the platform's VFS. Its own binary, as the shim becomes the process's default VFS. #![cfg(unix)] use notebook::{ - Replica, - session::{Event, Section}, + EditStatus, Error, Replica, + session::{Event, Section, SyncState}, +}; +use onestore::{ + PageCreation, + op::{Edit, Op, PageOp, SectionOp}, +}; +use std::{ + io, + path::{Path, PathBuf}, + sync::{MutexGuard, PoisonError, atomic::Ordering, mpsc}, + time::{Duration, Instant}, }; -use onestore::op::{Edit, Op, PageOp}; -use std::{path::Path, sync::atomic::Ordering, sync::mpsc, time::Duration}; #[path = "support/vfs.rs"] mod vfs; use vfs::{ - ARMED, KEEP_WAL, OPEN, SYNC_AFTER, SYNC_BEFORE, WAL_WRITES_AFTER_FAULT, WRITE_COMMIT, install, + ARMED, EXCLUSIVE, FAILING_OPENS, KEEP_WAL, SYNC_AFTER, SYNC_BEFORE, WAL_WRITES_AFTER_FAULT, + WRITE_COMMIT, install, }; +/// Holds the shim for one test, disarmed of what a failed test left. +fn exclusive() -> MutexGuard<'static, ()> { + install(); + let exclusive = EXCLUSIVE.lock().unwrap_or_else(PoisonError::into_inner); + ARMED.store(0, Ordering::SeqCst); + FAILING_OPENS.store(0, Ordering::SeqCst); + exclusive +} + +/// A section file in `root` and its cache, which the returned path names. +fn seed(root: &Path) -> PathBuf { + let source = onestore::create_section("Fault.one", "Original", "Fixture").unwrap(); + std::fs::write(root.join("Fault.one"), &source).unwrap(); + let cache = root.join("replica.sqlite"); + drop(Replica::create(&cache, &source).unwrap()); + cache +} + fn colour(space: onestore::ExGuid, colour: u32) -> Edit { Edit { at: 133_000_000_000_000_000 + u64::from(colour), @@ -25,17 +52,49 @@ fn colour(space: onestore::ExGuid, colour: u32) -> Edit { } } +/// The queued edits a cold open of `cache` reads, with the colour each gives the page. +fn colours(cache: &Path) -> Vec<(u64, u32)> { + Replica::open(cache) + .unwrap() + .pending() + .unwrap() + .iter() + .map(|edit| match &edit.edit.ops[..] { + [ + Op::Page { + op: PageOp::Color(Some(colour)), + .. + }, + ] => (edit.id, *colour), + ops => panic!("{ops:?}"), + }) + .collect() +} + +fn until(section: &Section, seen: impl Fn(&Event) -> bool) { + let deadline = Instant::now() + Duration::from_secs(10); + while !section.events().iter().any(&seen) { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn would_block(result: Result) -> bool { + matches!(result, Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) +} + /// What the section answered for the edit committed under `fault` and the one after it, /// whether a failed write was reported, and the queue as the section saw it. struct Outcome { replies: [Result; 2], failed: bool, - pending: Option>, + pending: Vec, } /// Seeds the WAL with one edit, so the armed commit's sync is the only one it makes, then -/// commits under `fault` and once more after it; with `crash`, leaves the cache open. -fn scenario(root: &Path, fault: u8, crash: bool) -> Outcome { +/// commits under `fault`, the cache failing to reopen `opens` times, and once more after it; +/// with `crash`, leaves the cache open. +fn scenario(root: &Path, fault: u8, opens: usize, crash: bool) -> Outcome { install(); let replica = Replica::open(root.join("replica.sqlite")).unwrap(); let section = Section::resume(root.join("Fault.one"), replica, true, || {}).unwrap(); @@ -54,9 +113,15 @@ fn scenario(root: &Path, fault: u8, crash: bool) -> Outcome { }; apply(1).unwrap(); WAL_WRITES_AFTER_FAULT.store(0, Ordering::SeqCst); + FAILING_OPENS.store(opens, Ordering::SeqCst); ARMED.store(fault, Ordering::SeqCst); let first = apply(2); assert_eq!(ARMED.load(Ordering::SeqCst), 0, "fault {fault} never fired"); + assert_eq!( + FAILING_OPENS.load(Ordering::SeqCst), + 0, + "the cache never reopened" + ); assert_eq!( WAL_WRITES_AFTER_FAULT.load(Ordering::SeqCst), 0, @@ -69,8 +134,10 @@ fn scenario(root: &Path, fault: u8, crash: bool) -> Outcome { .any(|event| matches!(event, Event::Failed(_))); let pending = section .pending() - .ok() - .map(|pending| pending.iter().map(|edit| edit.id).collect()); + .unwrap() + .iter() + .map(|edit| edit.id) + .collect(); if crash { std::mem::forget(section); } else { @@ -85,14 +152,15 @@ fn scenario(root: &Path, fault: u8, crash: bool) -> Outcome { #[test] fn commit_fault_child() { - let (Some(root), Some(fault)) = ( + let (Some(root), Some(fault), Some(opens)) = ( std::env::var_os("COMMIT_FAULT_CHILD"), std::env::var("COMMIT_FAULT").ok(), + std::env::var("COMMIT_FAULT_OPENS").ok(), ) else { return; }; let root = Path::new(&root); - let outcome = scenario(root, fault.parse().unwrap(), true); + let outcome = scenario(root, fault.parse().unwrap(), opens.parse().unwrap(), true); let ids = |result: &Result| result.as_ref().map_or(0, |id| *id); std::fs::write( root.join("outcome"), @@ -110,68 +178,491 @@ fn commit_fault_child() { #[test] fn a_failed_commit_is_decided_by_a_cold_read_and_never_replayed_twice() { - install(); - // (fault, whether the edit reached the cache, whether the cache stays closed) - for (fault, landed, closed) in [ - (SYNC_BEFORE, false, false), - (SYNC_AFTER, false, false), - (WRITE_COMMIT, false, false), - (SYNC_AFTER | KEEP_WAL, true, false), - (SYNC_BEFORE | OPEN, false, true), + let _exclusive = exclusive(); + // (fault, whether the edit reached the cache, how many reopens fail) + for (fault, landed, opens) in [ + (SYNC_BEFORE, false, 0), + (SYNC_AFTER, false, 0), + (WRITE_COMMIT, false, 0), + (SYNC_AFTER | KEEP_WAL, true, 0), + (SYNC_BEFORE, false, 2), + (SYNC_AFTER | KEEP_WAL, true, 2), ] { for crash in [false, true] { let directory = tempfile::tempdir().unwrap(); let root = directory.path(); - let source = onestore::create_section("Fault.one", "Original", "Fixture").unwrap(); - std::fs::write(root.join("Fault.one"), &source).unwrap(); - drop(Replica::create(root.join("replica.sqlite"), &source).unwrap()); + let cache = seed(root); let (first, second, failed) = if crash { let output = std::process::Command::new(std::env::current_exe().unwrap()) .args(["--exact", "commit_fault_child", "--nocapture"]) .env("COMMIT_FAULT_CHILD", root) .env("COMMIT_FAULT", fault.to_string()) + .env("COMMIT_FAULT_OPENS", opens.to_string()) .output() .unwrap(); assert!(output.status.success(), "{output:?}"); let outcome = std::fs::read_to_string(root.join("outcome")).unwrap(); let mut fields = outcome.split(' '); - let mut id = || Some(fields.next()?.parse::().unwrap()).filter(|id| *id > 0); + let mut id = || fields.next().unwrap().parse::().unwrap(); let (first, second) = (id(), id()); (first, second, fields.next() == Some("true")) } else { - let outcome = scenario(root, fault, false); - let [first, second] = outcome.replies; - if !closed { - let ids = [first.clone().unwrap(), second.clone().unwrap()]; - assert_eq!(&outcome.pending.unwrap()[1..], ids); - } else { - assert!(outcome.pending.is_none()); - } - (first.ok(), second.ok(), outcome.failed) + let outcome = scenario(root, fault, opens, false); + let ids = outcome.replies.map(Result::unwrap); + assert_eq!(outcome.pending[1..], ids); + (ids[0], ids[1], outcome.failed) }; - assert_eq!(failed, !landed, "fault {fault}"); - let cold = Replica::open(root.join("replica.sqlite")).unwrap(); - let colours: Vec<_> = cold - .pending() + // An edit waiting for the cache to reopen is reported, as one that did not land. + assert_eq!(failed, !landed || opens > 0, "fault {fault}, {opens} opens"); + assert_eq!( + colours(&cache)[1..], + [(first, 2), (second, 3)], + "fault {fault}, {opens} opens" + ); + } + } +} + +#[test] +fn edits_wait_while_the_cache_cannot_reopen_and_land_once() { + let _exclusive = exclusive(); + for close in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let cache = seed(root); + let section = Section::resume( + root.join("Fault.one"), + Replica::open(&cache).unwrap(), + true, + || {}, + ) + .unwrap(); + let space = section.pages().unwrap()[0].0; + let (replied, replies) = mpsc::channel(); + let apply = |value| { + let replied = replied.clone(); + section + .apply_with("Fixture", colour(space, value), move |result| { + replied + .send(result.map_err(|error| error.to_string())) + .unwrap(); + }) + .unwrap(); + }; + let reply = || { + replies + .recv_timeout(Duration::from_secs(10)) .unwrap() - .iter() - .map(|edit| match &edit.edit.ops[..] { - [ - Op::Page { - op: PageOp::Color(Some(colour)), - .. - }, - ] => (edit.id, *colour), - ops => panic!("{ops:?}"), + .unwrap() + }; + apply(1); + let seeded = reply(); + FAILING_OPENS.store(1000, Ordering::SeqCst); + ARMED.store(SYNC_BEFORE, Ordering::SeqCst); + apply(2); + until(§ion, |event| matches!(event, Event::Failed(_))); + // The cache is tried again once a second. + let left = FAILING_OPENS.load(Ordering::SeqCst); + std::thread::sleep(Duration::from_secs(3)); + let tries = left - FAILING_OPENS.load(Ordering::SeqCst); + // Never faster than once a second; as slow as the machine is busy. + assert!((1..=4).contains(&tries), "{tries} tries in 3 s"); + apply(3); + // A caller blocking on its edit learns at once, and the waiting edits stay. + assert!(would_block( + section.replica().apply("Fixture", colour(space, 4)) + )); + // A sync step meets the closed cache and the worker carries on. + section.wake(); + until( + §ion, + |event| matches!(event, Event::Failed(error) if error == "The cache reopens once it can"), + ); + // Each second the cache is tried again, and the edits still wait. + until(§ion, |event| matches!(event, Event::Failed(_))); + assert!(replies.try_recv().is_err()); + assert_eq!(section.page(space).unwrap().color, Some(3)); + assert!(would_block(section.written())); + assert!(would_block(section.pending())); + assert_eq!(notebook::unwritten(), 2); + let status = section.sync_status().unwrap(); + // The seed, queued before the commit that failed, and the two waiting. + assert_eq!((status.state(), status.queued), (SyncState::Failed, 3)); + // Nobody else takes the cache meanwhile, nor once the section closes. + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + let open = if close { + section.close().unwrap(); + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + None + } else { + Some(section) + }; + FAILING_OPENS.store(0, Ordering::SeqCst); + let ids = [reply(), reply()]; + if let Some(section) = open { + section.wake(); + until(§ion, |event| { + matches!(event, Event::Attempt { id, status: EditStatus::Published { .. } } + if *id == ids[1]) + }); + section.close().unwrap(); + let arena = onestore::Arena::default(); + let stored = + onestore::Section::open(&arena, std::fs::read(root.join("Fault.one")).unwrap()) + .unwrap(); + assert_eq!(stored.page(space).unwrap().color, Some(3)); + assert!(colours(&cache).is_empty()); + } else { + // Its cache is released once the edits are written. + let deadline = Instant::now() + Duration::from_secs(10); + while Replica::open(&cache).is_err() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(colours(&cache), [(seeded, 1), (ids[0], 2), (ids[1], 3)]); + } + assert_eq!(notebook::unwritten(), 0); + } +} + +#[test] +fn a_lost_cache_refuses_the_edits_that_wait() { + let _exclusive = exclusive(); + for corrupt in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let cache = seed(root); + let section = Section::resume( + root.join("Fault.one"), + Replica::open(&cache).unwrap(), + true, + || {}, + ) + .unwrap(); + let space = section.pages().unwrap()[0].0; + let (replied, replies) = mpsc::channel(); + let apply = |value| { + let replied = replied.clone(); + section + .apply_with("Fixture", colour(space, value), move |result| { + replied + .send(result.map_err(|error| error.to_string())) + .unwrap(); }) - .collect(); - if closed { - assert_eq!((first, second), (None, None)); - assert_eq!(colours.len(), 1, "fault {fault}"); - } else { - let (first, second) = (first.unwrap(), second.unwrap()); - assert_eq!(&colours[1..], [(first, 2), (second, 3)], "fault {fault}"); + .unwrap(); + }; + let reply = || replies.recv_timeout(Duration::from_secs(10)).unwrap(); + apply(1); + reply().unwrap(); + FAILING_OPENS.store(usize::MAX, Ordering::SeqCst); + ARMED.store(SYNC_BEFORE, Ordering::SeqCst); + apply(2); + until(§ion, |event| matches!(event, Event::Failed(_))); + for suffix in ["-wal", "-shm"] { + let mut file = cache.clone().into_os_string(); + file.push(suffix); + let _ = std::fs::remove_file(file); + } + if corrupt { + std::fs::write(&cache, [0xab; 4096]).unwrap(); + } else { + std::fs::remove_file(&cache).unwrap(); + } + FAILING_OPENS.store(0, Ordering::SeqCst); + assert!(reply().is_err()); + assert_eq!(notebook::unwritten(), 0); + let pending = section.pending(); + assert!(pending.is_err() && !would_block(pending)); + apply(3); + assert!(reply().is_err()); + let status = section.sync_status().unwrap(); + assert_eq!((status.state(), status.queued), (SyncState::Failed, 0)); + // The sync worker meets the lost cache and stops. + std::thread::sleep(Duration::from_millis(100)); + section.events(); + section.wake(); + until(§ion, |event| matches!(event, Event::Failed(_))); + assert!(section.close().is_err()); + } +} + +#[test] +fn a_closed_cache_stays_owned_and_its_landed_commit_counts_once() { + let _exclusive = exclusive(); + for close in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let cache = seed(root); + let section = Section::resume( + root.join("Fault.one"), + Replica::open(&cache).unwrap(), + true, + || {}, + ) + .unwrap(); + let space = section.pages().unwrap()[0].0; + let (replied, replies) = mpsc::channel::>(); + let reply = || { + replies + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap() + }; + let apply = |value| { + let replied = replied.clone(); + section + .apply_with("Fixture", colour(space, value), move |result| { + replied + .send(result.map_err(|error| error.to_string())) + .unwrap(); + }) + .unwrap(); + }; + apply(1); + let seeded = reply(); + FAILING_OPENS.store(usize::MAX, Ordering::SeqCst); + ARMED.store(SYNC_AFTER | KEEP_WAL, Ordering::SeqCst); + // A caller blocking on an edit whose commit fails hears at once, though it landed. + let asked = Instant::now(); + assert!( + section + .replica() + .apply("Fixture", colour(space, 2)) + .is_err() + ); + assert!(asked.elapsed() < Duration::from_secs(5)); + // Waits after the commit the closed cache holds, in the same batch. + apply(3); + until(§ion, |event| matches!(event, Event::Failed(_))); + // Nobody else takes the cache to publish and acknowledge that commit meanwhile. + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + let open = if close { + section.close().unwrap(); + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + None + } else { + Some(section) + }; + FAILING_OPENS.store(0, Ordering::SeqCst); + let third = reply(); + if let Some(section) = open { + // Reread once the cache reopens, the section is the queue. + let deadline = Instant::now() + Duration::from_secs(10); + while !section.pending().is_ok_and(|pending| pending.len() == 3) { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(section.page(space).unwrap().color, Some(3)); + section.close().unwrap(); + } else { + let deadline = Instant::now() + Duration::from_secs(10); + while Replica::open(&cache).is_err() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + } + let stored = colours(&cache); + assert_eq!(stored.len(), 3, "{stored:?}"); + assert_eq!( + (stored[0], stored[1].1, stored[2]), + ((seeded, 1), 2, (third, 3)) + ); + } +} + +#[test] +fn closed_crash_child() { + let (Some(root), Some(fault)) = ( + std::env::var_os("CLOSED_CRASH_CHILD"), + std::env::var("COMMIT_FAULT").ok(), + ) else { + return; + }; + install(); + let root = Path::new(&root); + let replica = Replica::open(root.join("replica.sqlite")).unwrap(); + let section = Section::resume(root.join("Fault.one"), replica, true, || {}).unwrap(); + let space = section.pages().unwrap()[0].0; + let (replied, replies) = mpsc::channel(); + let replied = move |result: Result| replied.send(result.is_ok()).unwrap(); + let apply = |value, replied: Box) + Send>| { + section + .apply_with("Fixture", colour(space, value), replied) + .unwrap(); + }; + apply(1, Box::new(replied)); + assert!(replies.recv_timeout(Duration::from_secs(10)).unwrap()); + FAILING_OPENS.store(usize::MAX, Ordering::SeqCst); + ARMED.store(fault.parse().unwrap(), Ordering::SeqCst); + apply(2, Box::new(drop)); + until(§ion, |event| matches!(event, Event::Failed(_))); + apply(3, Box::new(drop)); + std::thread::sleep(Duration::from_millis(100)); + // Ends as a crash would, the cache still closed. + std::process::exit(0); +} + +#[test] +fn a_crash_while_the_cache_is_closed_leaves_what_landed() { + for (fault, landed) in [(SYNC_BEFORE, false), (SYNC_AFTER | KEEP_WAL, true)] { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let cache = seed(root); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "closed_crash_child", "--nocapture"]) + .env("CLOSED_CRASH_CHILD", root) + .env("COMMIT_FAULT", fault.to_string()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + let colours: Vec<_> = colours(&cache) + .into_iter() + .map(|(_, colour)| colour) + .collect(); + assert_eq!( + colours, + if landed { vec![1, 2] } else { vec![1] }, + "fault {fault}" + ); + } +} + +/// A section, offline, over a seeded cache in `root`, with its first page's space. +fn opened(root: &Path) -> (PathBuf, Section, onestore::ExGuid) { + let cache = seed(root); + let section = Section::resume( + root.join("Fault.one"), + Replica::open(&cache).unwrap(), + true, + || {}, + ) + .unwrap(); + let space = section.pages().unwrap()[0].0; + (cache, section, space) +} + +#[test] +fn edits_wait_for_the_reread_a_refused_edit_needs_while_the_cache_is_closed() { + let _exclusive = exclusive(); + for close in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let (cache, section, space) = opened(root); + let (replied, replies) = mpsc::channel::<(u32, Result)>(); + let apply = |space, value| { + let replied = replied.clone(); + section + .apply_with("Fixture", colour(space, value), move |result| { + let result = result.map_err(|error| error.to_string()); + replied.send((value, result)).unwrap(); + }) + .unwrap(); + }; + let reply = || replies.recv_timeout(Duration::from_secs(10)).unwrap(); + apply(space, 1); + let seeded = reply().1.unwrap(); + FAILING_OPENS.store(usize::MAX, Ordering::SeqCst); + ARMED.store(SYNC_BEFORE, Ordering::SeqCst); + // A page a caller blocks on is refused, its commit never landing. + let creation = PageCreation::new(None, Some("Refused"), "Fixture").unwrap(); + let made = creation.space(); + let create = Edit { + at: 133_000_000_000_000_100, + ops: vec![Op::Section(SectionOp::Create(creation))], + }; + assert!(section.replica().apply("Fixture", create).is_err()); + // Until the cache reopens to reread the section, edits wait to apply, one on the + // refused page among them. + apply(made, 2); + apply(space, 3); + std::thread::sleep(Duration::from_millis(1500)); + assert!(replies.try_recv().is_err()); + assert_eq!(section.page(space).unwrap().color, Some(1)); + assert!(would_block(section.written())); + assert_eq!(notebook::unwritten(), 2); + let open = if close { + section.close().unwrap(); + None + } else { + Some(section) + }; + FAILING_OPENS.store(0, Ordering::SeqCst); + // A host reading the section steadily, as one typing does, keeps no retry back. + let deadline = Instant::now() + Duration::from_secs(10); + let mut answered = Vec::new(); + while answered.len() < 2 { + assert!(Instant::now() < deadline); + if let Some(section) = &open { + let _ = section.page(space); } + answered.extend(replies.recv_timeout(Duration::from_millis(300))); } + let mut answered: [_; 2] = answered.try_into().unwrap(); + answered.sort_by_key(|(value, _)| *value); + let [(_, dependent), (_, kept)] = answered; + assert!(dependent.is_err(), "{dependent:?}"); + let kept = kept.unwrap(); + if let Some(section) = open { + // The reread tells the host to reload what it shows. + until( + §ion, + |event| matches!(event, Event::Changed(spaces) if spaces.contains(&made)), + ); + assert!( + section + .pages() + .unwrap() + .iter() + .all(|(listed, ..)| *listed != made) + ); + assert_eq!(section.page(space).unwrap().color, Some(3)); + section.close().unwrap(); + } + let deadline = Instant::now() + Duration::from_secs(10); + while Replica::open(&cache).is_err_and(|error| error.busy()) { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(colours(&cache), [(seeded, 1), (kept, 3)]); } } + +#[test] +fn a_cache_whose_owner_lock_cannot_be_taken_stays_held_by_its_warm_connection() { + let _exclusive = exclusive(); + let directory = tempfile::tempdir().unwrap(); + let root = directory.path(); + let (cache, section, space) = opened(root); + let (replied, replies) = mpsc::channel(); + let apply = |value| { + let replied = replied.clone(); + section + .apply_with("Fixture", colour(space, value), move |result| { + replied + .send(result.map_err(|error| error.to_string())) + .unwrap(); + }) + .unwrap(); + }; + let reply = || { + replies + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap() + }; + apply(1); + let seeded = reply(); + let mut lock = cache.clone().into_os_string(); + lock.push("-lock"); + std::fs::create_dir(&lock).unwrap(); + ARMED.store(SYNC_BEFORE, Ordering::SeqCst); + apply(2); + until(§ion, |event| matches!(event, Event::Failed(_))); + std::thread::sleep(Duration::from_millis(1500)); + assert!(replies.try_recv().is_err()); + assert!(Replica::open(&cache).is_err_and(|error| error.busy())); + std::fs::remove_dir(&lock).unwrap(); + let second = reply(); + section.close().unwrap(); + assert_eq!(colours(&cache), [(seeded, 1), (second, 2)]); +} diff --git a/crates/notebook/tests/support/vfs.rs b/crates/notebook/tests/support/vfs.rs index 0af3c4f05363d11095277c9d3ac5c5f745eb0b5a..5a6b9ca2269d612bfc6c631618241f8cbb783d79 100644 --- a/crates/notebook/tests/support/vfs.rs +++ b/crates/notebook/tests/support/vfs.rs @@ -1,6 +1,6 @@ //! A VFS over the platform's, made the process's default by `install`, that fails what a -//! test arms: once, the WAL's sync or commit frame or the next database open; or every WAL -//! write of the database `FAILING` names, for as long as it names it. +//! test arms: once, the WAL's sync or commit frame; the replica's next `FAILING_OPENS` opens; +//! or every WAL write of the database `FAILING` names, for as long as it names it. #![allow(dead_code)] use rusqlite::ffi; @@ -19,8 +19,6 @@ pub const SYNC_BEFORE: u8 = 1; pub const SYNC_AFTER: u8 = 2; /// The commit frame's header is never written. pub const WRITE_COMMIT: u8 = 4; -/// The next open of a database fails, as the cold reopen after a failed commit. -pub const OPEN: u8 = 8; /// After a fault, the next sync fails, as the checkpoint closing the warm connection makes, /// which keeps the WAL. pub const KEEP_WAL: u8 = 16; @@ -29,12 +27,15 @@ pub static ARMED: AtomicU8 = AtomicU8::new(0); /// Between a fault and the next database open, as the replica reopens the cache cold. static FIRED: AtomicBool = AtomicBool::new(false); pub static WAL_WRITES_AFTER_FAULT: AtomicUsize = AtomicUsize::new(0); +/// How many database opens on a replica's own threads still fail, as the cold reopen after a +/// failed commit can, of the database `FAILING` names where it names one; a test's opens never do. +pub static FAILING_OPENS: AtomicUsize = AtomicUsize::new(0); static REAL: OnceLock = OnceLock::new(); /// The platform's WAL file methods, the shim's copy of them, and the copy failing every write. static WAL: OnceLock<(usize, usize, usize)> = OnceLock::new(); /// While set, every write to this database's WAL fails, for WALs opened while it was set. pub static FAILING: Mutex> = Mutex::new(None); -/// Held by a test for as long as it sets `FAILING`, which one test sets at a time. +/// Held by a test for as long as it arms a fault, which one test does at a time. pub static EXCLUSIVE: Mutex<()> = Mutex::new(()); fn fire(fault: u8) -> bool { @@ -58,8 +59,24 @@ unsafe extern "C" fn open( ) -> c_int { if flags & ffi::SQLITE_OPEN_MAIN_DB != 0 { FIRED.store(false, Ordering::SeqCst); - if fire(OPEN) { - FIRED.store(false, Ordering::SeqCst); + let named = unsafe { CStr::from_ptr(name) } + .to_string_lossy() + .into_owned(); + let chosen = FAILING + .lock() + .unwrap() + .as_ref() + .is_none_or(|database| named.starts_with(&*database.to_string_lossy())); + if chosen + && std::thread::current() + .name() + .is_some_and(|name| name.starts_with("onestore-")) + && FAILING_OPENS + .try_update(Ordering::SeqCst, Ordering::SeqCst, |opens| { + opens.checked_sub(1) + }) + .is_ok() + { unsafe { (*file).pMethods = std::ptr::null() }; return ffi::SQLITE_CANTOPEN; } diff --git a/crates/snowbound/src/manage.rs b/crates/snowbound/src/manage.rs index 93bb8d1c3c471c01d679136318ec7f34a6cd9d0e..bd41c990c7b5a6de5aca8f5d27f0313111409331 100644 --- a/crates/snowbound/src/manage.rs +++ b/crates/snowbound/src/manage.rs @@ -436,12 +436,7 @@ impl State { state.vanishing.remove(&location); let waiting = (state.session.as_ref()) .filter(|session| session.library.location == location) - .is_some_and(|session| { - session - .section - .pending() - .is_ok_and(|pending| !pending.is_empty()) - }); + .is_some_and(|session| unsaved(&session.section) > 0); let listed = (state.notebooks.iter()).find(|open| open.location == location); if let Some(library) = listed.cloned().filter(|_| gone && !waiting) { state.close_notebook(&library); @@ -583,10 +578,8 @@ impl State { .filter(|library| library.in_icloud()) .cloned() .collect(); - let waiting = (self.sessions()).filter(|session| { - session.library.in_icloud() - && (session.section.pending()).is_ok_and(|pending| !pending.is_empty()) - }); + let waiting = (self.sessions()) + .filter(|session| session.library.in_icloud() && unsaved(&session.section) > 0); for session in waiting { let Some(folder) = platform::settings_dir().map(|folder| folder.join("Recovery")) else { @@ -653,8 +646,7 @@ impl State { self.persist().unwrap_or_else(|error| eprintln!("{error}")); let open = (self.session.as_ref()) .filter(|session| Arc::ptr_eq(&session.library, &library)) - .and_then(|session| session.section.pending().ok()) - .map_or(0, |pending| pending.len() as u64); + .map_or(0, |session| unsaved(&session.section)); let waiting = (library.background.iter()) .flat_map(|background| background.status()) .map(|(_, sync)| sync.queued) @@ -1211,6 +1203,12 @@ fn art_ops(page: &Page, art: Vec) -> Vec { ops } +/// The changes `section` holds that its file does not yet, as many as can be told, and at least +/// one where none can. +fn unsaved(section: ¬ebook::session::Section) -> u64 { + section.sync_status().map_or(1, |sync| sync.queued) +} + #[cfg(test)] mod tests { use super::*; @@ -1225,6 +1223,53 @@ mod tests { creation.dated(&date, &time).unwrap() } + #[test] + fn unsaved_counts_what_a_closed_cache_holds() { + use std::sync::atomic::Ordering; + crate::vfs::install(); + let _exclusive = + (crate::vfs::EXCLUSIVE.lock()).unwrap_or_else(std::sync::PoisonError::into_inner); + let temporary = + std::env::temp_dir().join(format!("snowbound-unsaved-{}", std::process::id())); + let _ = notebook::fs::remove_dir_all(&temporary); + notebook::fs::create_dir_all(&temporary).unwrap(); + let file = temporary.join("Unsaved.one"); + let cache = temporary.join("unsaved.sqlite"); + let source = onestore::create_section("Unsaved.one", "Original", AUTHOR).unwrap(); + notebook::fs::write(&file, &source).unwrap(); + let open = || { + let replica = notebook::Replica::open(&cache).unwrap(); + notebook::session::Section::resume(&file, replica, true, || {}).unwrap() + }; + drop(notebook::Replica::create(&cache, &source).unwrap()); + let section = open(); + let space = section.pages().unwrap()[0].0; + let colour = |colour| Edit { + at: crate::filetime(), + ops: vec![Op::Page { + space, + op: PageOp::Color(Some(colour)), + }], + }; + section.replica().apply(AUTHOR, colour(1)).unwrap(); + section.close().unwrap(); + // The cache's next commit fails, and it cannot reopen; the edit is refused. + *crate::vfs::FAILING.lock().unwrap() = Some(cache.canonicalize().unwrap()); + crate::vfs::FAILING_OPENS.store(usize::MAX, Ordering::SeqCst); + let section = open(); + assert_eq!(unsaved(§ion), 1); + assert!(section.replica().apply(AUTHOR, colour(2)).is_err()); + assert!(section.pending().is_err()); + assert_eq!(unsaved(§ion), 1); + *crate::vfs::FAILING.lock().unwrap() = None; + crate::vfs::FAILING_OPENS.store(0, Ordering::SeqCst); + section.close().unwrap(); + while notebook::Replica::open(&cache).is_err() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let _ = notebook::fs::remove_dir_all(&temporary); + } + #[test] fn repeated_section_renames_follow_identity_and_keep_the_latest_label() { let temporary =