diff --git a/apps/ios/Snowbound/ICloud.swift b/apps/ios/Snowbound/ICloud.swift index 138ba291f91a0c50590ca6fdb577093777fc8dbc..11c6d3d6d704b921952c9ec7944bf9a438af98e2 100644 --- a/apps/ios/Snowbound/ICloud.swift +++ b/apps/ios/Snowbound/ICloud.swift @@ -1,4 +1,4 @@ -import Foundation +import UIKit /// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another /// device's commit that lost; the library merges each into the file. @@ -58,10 +58,11 @@ enum ICloud { /// Whether iCloud is signed in with iCloud Drive on, container or not. static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil } - /// Has the library merge conflict versions, follows the account, and hears of notebooks - /// another device adds or removes; call once at launch. + /// Has the library merge conflict versions and this device's uploads iCloud Drive lost, + /// follows the account, and hears of notebooks another device adds or removes; call once + /// at launch. static func start() { - sb_set_versions(listVersions, retireVersion) + sb_set_versions(listVersions, retireVersion, UIDevice.current.name) if automated { return lookUp() } NotificationCenter.default.addObserver( forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main diff --git a/arc/sync.md b/arc/sync.md index 3b61adcbac690ff5b96a0b4a9c9420db60e1baec..ebd3e500b0dd44b5951351f5a049b81ce0a31e28 100644 --- a/arc/sync.md +++ b/arc/sync.md @@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai device that wrote it. A version of another section, or one that cannot be read, is kept beside the file as `Name (Device).one` instead. +iCloud Drive can also lose an upload outright: another device's lands over it and no conflict +version reaches this one. So on the iPhone and on Android each image a section publishes to a +file a provider keeps stays in the cache (`notebook::published`) until the file is seen +otherwise, then lists as one more version: one the file holds merges as nothing and goes, one +it lost merges back as a conflict version does, its conflict pages named for this device. One +that cannot merge is kept beside the file as `Name (Device).one` on the iPhone, and in the +cache on Android, where no provider takes a new document beside it. + On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted (a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists diff --git a/crates/mobile/include/snowbound.h b/crates/mobile/include/snowbound.h index 7b850d304925fff40e36432a70956b73060e41ba..8cff4b8473ca1b3386d64866d89d8c9a79ce5091 100644 --- a/crates/mobile/include/snowbound.h +++ b/crates/mobile/include/snowbound.h @@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator); void sb_set_sync_wake(void (*wake)(void)); typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context); typedef bool (*sb_retire)(const char *path, const char *id, bool keep); -void sb_set_versions(sb_versions versions, sb_retire retire); +void sb_set_versions(sb_versions versions, sb_retire retire, const char *device); char *sb_tags(void); char *sb_pens(uint32_t section); bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba); diff --git a/crates/mobile/src/library.rs b/crates/mobile/src/library.rs index f4e8dc16f8adedab85cba18eaff07d780c55a2ce..ba9ca405b364cd4899df23125c514fda962dc116 100644 --- a/crates/mobile/src/library.rs +++ b/crates/mobile/src/library.rs @@ -9,6 +9,7 @@ use notebook::{ discover::{Folder, Reason, SectionState}, live::{self, Hello, Reach, share::Guest}, provider::Documents, + published::{Published, Written}, session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus}, smb::{Client, Credentials}, }; @@ -103,28 +104,60 @@ pub type Versions = extern "C" fn( /// beside the file when `keep`; false when it could not. pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool; -static VERSIONS: OnceLock<(Versions, Retire)> = OnceLock::new(); +static VERSIONS: OnceLock<(Versions, Retire, String)> = OnceLock::new(); /// Merges the conflict versions the host lists into their files from now on, then has the -/// host retire them; set once, before any library opens. +/// host retire them, and merges back this device's publications a file provider lost, which +/// conflict pages name for `device`; set once, before any library opens. +/// +/// # Safety +/// `device` is NUL-terminated UTF-8. #[unsafe(no_mangle)] -pub extern "C" fn sb_set_versions(versions: Versions, retire: Retire) { - let _ = VERSIONS.set((versions, retire)); +pub unsafe extern "C" fn sb_set_versions( + versions: Versions, + retire: Retire, + device: *const c_char, +) { + let _ = VERSIONS.set((versions, retire, string(device))); } -/// A local section file read and published under the host's file coordination. -struct Coordinated(PathBuf); +/// A local section file read and published under the host's file coordination. One a file +/// provider keeps elsewhere, which syncs it whole with no lock, keeps each image this device +/// publishes until the file holds it, and merges one another device's upload replaced back +/// (`notebook::published`). +struct Coordinated { + file: PathBuf, + published: Option, +} + +impl Coordinated { + fn new(file: &Path, cache: &Path, elsewhere: bool) -> Self { + let published = elsewhere + .then(|| { + notebook::location::claimed(cache, ¬ebook::location::local(file).ok()?).ok() + }) + .flatten() + .map(|folder| { + let device = VERSIONS.get().map(|(.., device)| device.clone()); + Published::new(folder.join("published"), device) + }); + Self { + file: file.to_owned(), + published, + } + } +} impl Remote for Coordinated { fn read(&mut self) -> io::Result> { - coordinated(&self.0, false, || onestore::read_file(&self.0))? + coordinated(&self.file, false, || onestore::read_file(&self.file))? } /// The header and length, as the file remote's uncoordinated change detector reads them. fn stamp(&mut self) -> io::Result { - coordinated(&self.0, false, || { + coordinated(&self.file, false, || { use std::io::Read; - let mut file = std::fs::File::open(&self.0)?; + let mut file = std::fs::File::open(&self.file)?; let mut header = [0; 1024]; file.read_exact(&mut header)?; Ok(Stamp { @@ -135,28 +168,48 @@ impl Remote for Coordinated { } fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> { - coordinated(&self.0, true, || transaction.commit_file(&self.0)).map_err(uncommitted)? + coordinated(&self.file, true, || { + transaction.commit_file(&self.file)?; + if let Some(published) = &self.published { + // A copy it could not keep only leaves the publication unprotected. + let _ = std::fs::File::open(&self.file) + .and_then(|mut file| published.keep(Written::InPlace(&mut file), transaction)); + } + Ok(()) + }) + .map_err(uncommitted)? } fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> { - coordinated(&self.0, true, || onestore::confirm_file(&self.0, base)).map_err(uncommitted)? + coordinated(&self.file, true, || { + onestore::confirm_file(&self.file, base) + }) + .map_err(uncommitted)? } + /// The host's conflict versions, then this device's publications the file lost. fn versions(&mut self) -> io::Result> { - let Some((versions, _)) = VERSIONS.get() else { - return Ok(Vec::new()); - }; - extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) { - // SAFETY: `context` is the list below, alive for the host's call. - let listed = unsafe { &mut *context.cast::>() }; - listed.push(notebook::Version { - id: string(id), - device: (!device.is_null()).then(|| string(device)), - }); - } let mut listed: Vec = Vec::new(); - let path = CString::new(self.0.as_os_str().as_encoded_bytes())?; - versions(path.as_ptr(), found, (&raw mut listed).cast()); + if let Some((versions, ..)) = VERSIONS.get() { + extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) { + // SAFETY: `context` is the list below, alive for the host's call. + let listed = unsafe { &mut *context.cast::>() }; + listed.push(notebook::Version { + id: string(id), + device: (!device.is_null()).then(|| string(device)), + }); + } + let path = CString::new(self.file.as_os_str().as_encoded_bytes())?; + versions(path.as_ptr(), found, (&raw mut listed).cast()); + } + if self.published.is_some() { + let stamp = self.stamp()?; + listed.extend( + self.published + .iter() + .flat_map(|published| published.versions(&stamp)), + ); + } Ok(listed) } @@ -165,11 +218,31 @@ impl Remote for Coordinated { std::fs::read(id) } + /// A publication of this device's that cannot merge is kept whole beside the file, as + /// the host keeps a conflict version: "Name (Device).one". fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { - let Some((_, retire)) = VERSIONS.get() else { + if let Some(published) = &self.published + && published.holds(id) + { + if keep { + let stem = self.file.file_stem().unwrap_or_default().to_string_lossy(); + let device = VERSIONS.get().map_or("this device", |(.., device)| device); + let beside = (1..) + .map(|n| match n { + 1 => format!("{stem} ({device}).one"), + _ => format!("{stem} ({device} {n}).one"), + }) + .map(|name| self.file.with_file_name(name)) + .find(|beside| std::fs::symlink_metadata(beside).is_err()) + .ok_or(io::ErrorKind::AlreadyExists)?; + coordinated(&beside, true, || std::fs::copy(id, &beside))??; + } + return std::fs::remove_file(id); + } + let Some((_, retire, _)) = VERSIONS.get() else { return Ok(()); }; - let path = CString::new(self.0.as_os_str().as_encoded_bytes())?; + let path = CString::new(self.file.as_os_str().as_encoded_bytes())?; let id = CString::new(id)?; if retire(path.as_ptr(), id.as_ptr(), keep) { Ok(()) @@ -297,10 +370,10 @@ pub struct Library { offline: AtomicBool, /// Keys of the protected sections unlocked this run, by catalog path; in memory only. keys: Mutex>, - /// How long edits wait for a pause in typing before they publish (`Section::set_pause`): - /// a file provider such as iCloud Drive uploads each publication, and makes one concurrent - /// with another device's a conflict version. - pause: Duration, + /// A file provider keeps the notebook elsewhere too, as iCloud Drive does: edits wait for a + /// pause in typing before they publish (`Section::set_pause`), as it uploads each + /// publication and makes one concurrent with another device's a conflict version. + elsewhere: bool, } /// The pause edits to a file a provider keeps elsewhere wait for. @@ -400,10 +473,11 @@ impl Library { (None, Place::File(path.to_owned()), None) } else { let mut notebook = Notebook::open(path, cache)?; + let kept = cache.to_owned(); let background = notebook.background_with( local, !local, - |file| Coordinated(file.to_owned()), + move |file| Coordinated::new(file, &kept, !local), sync_woken, )?; background.watch(notebook.replicas()); @@ -422,7 +496,7 @@ impl Library { open: Mutex::default(), offline: AtomicBool::new(false), keys: Default::default(), - pause: if local { Duration::ZERO } else { PAUSE }, + elsewhere: !local, }) } @@ -445,7 +519,7 @@ impl Library { open: Mutex::default(), offline: AtomicBool::new(false), keys: Default::default(), - pause: Duration::ZERO, + elsewhere: false, }; let reached = library.with_notebook(false, |_| Ok(())); match library.listed() { @@ -504,7 +578,7 @@ impl Library { open: Mutex::default(), offline: AtomicBool::new(offline), keys: Default::default(), - pause: Duration::ZERO, + elsewhere: false, }) } @@ -541,7 +615,7 @@ impl Library { open: Mutex::default(), offline: AtomicBool::new(false), keys: Default::default(), - pause: if local { Duration::ZERO } else { PAUSE }, + elsewhere: !local, }) } @@ -783,7 +857,11 @@ impl Library { } }; section.set_offline(self.offline.load(Ordering::Relaxed)); - section.set_pause(self.pause); + section.set_pause(if self.elsewhere { + PAUSE + } else { + Duration::ZERO + }); if let Some(background) = &self.background { background.hold(path, §ion); } @@ -796,18 +874,19 @@ impl Library { notify: impl Fn() + Send + Sync + 'static, ) -> Result { let offline = self.offline.load(Ordering::Relaxed); + let (cache, elsewhere) = (self.cache.clone(), self.elsewhere); + let connect = move |file: &Path| Ok(Coordinated::new(file, &cache, elsewhere)); match &self.place { Place::File(file) => Ok(session::Section::open_with( file, &self.cache, - |file| Ok(Coordinated(file.to_owned())), + connect, offline, notify, )?), Place::Folder(_) => { let notebook = self.notebook(); let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?; - let connect = |file: &Path| Ok(Coordinated(file.to_owned())); Ok(match self.unlocked().get(path) { Some(key) => { notebook.section_unlocked_with(path, key, connect, offline, notify)? diff --git a/crates/mobile/src/tests.rs b/crates/mobile/src/tests.rs index 7153afe8f3cc8c4bcf2b14414077b426028c0776..39f2189c6a11a33b65c90a8c7674451880c1e99e 100644 --- a/crates/mobile/src/tests.rs +++ b/crates/mobile/src/tests.rs @@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String { shown } +/// The titles and text of every page and conflict page each section file in `root` stores. +fn stored(root: &Path) -> String { + let mut text = String::new(); + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().is_some_and(|extension| extension == "one") { + let arena = onestore::Arena::default(); + let mut section = + onestore::Section::open(&arena, std::fs::read(path).unwrap()).unwrap(); + let conflicts = section.conflicts().unwrap().into_iter(); + let spaces = (section + .pages() + .unwrap() + .into_iter() + .map(|(space, ..)| space)) + .chain( + conflicts + .flat_map(|(_, pages)| pages) + .map(|page| page.space), + ); + for space in spaces.collect::>() { + let page = section.page(space).unwrap(); + text += &format!("{}\n{}\n", page.title, canvas::search::page_text(&page)); + } + } + } + text +} + /// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy: /// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone -/// types on it, and a section the phone adds. The Mac's upload wins, so the phone's next -/// launch finds its own file a conflict version, and opens, lists and merges it through the C -/// surface as iOS does, its pages open and typed on in the Reading View all the while. -/// Nothing either device typed goes missing, merged or set aside whole. +/// types on it, and a section the phone adds. The Mac's upload wins: iCloud Drive keeps the +/// phone's file as a conflict version, or loses it. The phone's next launch opens, lists and +/// merges it back through the C surface as iOS does, its pages open and typed on in the +/// Reading View all the while. Nothing either device typed goes missing, merged or set aside +/// whole. #[test] -fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { - library::sb_set_versions(cloud_versions, cloud_retire); - // Whether the devices' copies already share an edit of the page both type on. - for shared in [false, true] { +fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { + unsafe { + library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr()) + }; + // Whether the devices' copies already share an edit of the page both type on, and + // whether iCloud Drive lost the phone's upload rather than keep it as a version. + for (shared, lost) in [(false, false), (true, false), (false, true), (true, true)] { let directory = tempfile::tempdir().unwrap(); let folder = |name: &str| directory.path().join(name); let (phone, mac) = (folder("phone"), folder("mac")); @@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { let version = folder("version.one"); std::fs::rename(&file, &version).unwrap(); std::fs::copy(mac.join(&path), &file).unwrap(); - let device = "Clover’s iPhone".to_owned(); - CLOUD_VERSIONS.lock().unwrap().push(( - std::fs::canonicalize(&file).unwrap(), - version, - device, - )); + if !lost { + let device = "Clover’s iPhone".to_owned(); + let file = std::fs::canonicalize(&file).unwrap(); + CLOUD_VERSIONS.lock().unwrap().push((file, version, device)); + } // The launch: the notebook opens and lists, the page shown last opens, and the // background merges the version meanwhile. @@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { .collect(); sb_library_sync_now(&library); let deadline = Instant::now() + Duration::from_secs(10); - while !CLOUD_VERSIONS.lock().unwrap().is_empty() { - assert!(Instant::now() < deadline, "the version never merged"); + while !CLOUD_VERSIONS.lock().unwrap().is_empty() + || !stored(&phone).contains("Phone first words") + { + assert!(Instant::now() < deadline, "the phone's upload never merged"); library::sb_section_poll(§ion, &mut 0); for view in &mut views { unsafe { sb_insert(view, c"Launch ".as_ptr()) }; diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 94afba057499f4e6613a07de864fc7149526b61c..ea5e1dbb3a7b01ad1486f6eb88f7fea17a53247f 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg (`Section::seal_as`), so a version merged once, here or on another device, merges as nothing again. It is published on the file's stamp, then retired; one of another section, or one that cannot be read, is retired with `keep` for the remote to keep beside the file. +A provider can also lose an upload outright, another device's landing over it with no version +kept. A remote keeps what it publishes with `published::Published` (`keep` after each +publication) and lists each image the file no longer stands as among its versions +(`Published::versions`): one the file holds merges as nothing and is dropped, one it lost +merges back. Publication attempts are recorded before network I/O. An attempt confirms only when the remote holds its revisions, or every page it changed as it changed them, diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index c2873ef398a4aabc367daee53b91ec54579b5156..f28cf69b8648d81ff2d9fc998ca55f6ddaf43a10 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -33,6 +33,7 @@ mod merge; mod migrate; pub mod package; pub mod provider; +pub mod published; mod queue; mod recovery; mod resolve; diff --git a/crates/notebook/src/location.rs b/crates/notebook/src/location.rs index f72a9adb1c1f92b5b73e63bce8f603c08ab21d56..f56575f7afce81b9e4461fa11b414344607a3bf1 100644 --- a/crates/notebook/src/location.rs +++ b/crates/notebook/src/location.rs @@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> { } /// The folder for `location`, made and labelled with it where it was not. -fn claimed(cache: &Path, location: &str) -> io::Result { +pub fn claimed(cache: &Path, location: &str) -> io::Result { let folder = folder(cache, location); let label = folder.join(LOCATION); if fs::metadata(&label).is_err() { diff --git a/crates/notebook/src/provider.rs b/crates/notebook/src/provider.rs index a89db5a66f58865c1474c3b0ada1c8f9ec8effd7..3f5525661a7c35c7b12dfd9a744768bb1da968c3 100644 --- a/crates/notebook/src/provider.rs +++ b/crates/notebook/src/provider.rs @@ -4,7 +4,11 @@ //! notebook root (`session::Notebook::open_provider`), a picked document a lone section //! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache. -use crate::{Error, Result, discover, session::Storage}; +use crate::{ + Error, Result, discover, + published::{Published, Written}, + session::Storage, +}; use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction}; use std::{ collections::BTreeMap, @@ -12,7 +16,6 @@ use std::{ io::{self, Read, Seek, SeekFrom, Write}, path::PathBuf, sync::{Arc, Mutex, MutexGuard, PoisonError}, - time::Duration, }; /// A document as its provider describes it. @@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result> { Ok(bytes) } -/// What a write wrote: the document in place, or its whole image. -enum Written<'a> { - InPlace(&'a mut File), - Image(&'a [u8]), -} - /// A document as a commit writes it: its seekable descriptor, or its image to write whole. enum Io { File(File), @@ -351,58 +348,12 @@ impl CommitIo for Io { } } -/// The hex of a stamp's digest, which names an image this device keeps. -fn digest(stamp: &Stamp) -> String { - use sha2::Digest; - let digest = sha2::Sha256::new() - .chain_update(stamp.header) - .chain_update(stamp.length.to_le_bytes()) - .finalize(); - hex(&digest[..16]) -} - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} - -fn file_of(header: &[u8]) -> Option<[u8; 16]> { - Some(onestore::Header::parse(header).ok()?.file_id) -} - -/// An image this device keeps, as its name gives it: `--.one`, the -/// document's identity, then the digests of the image's stamp and of the stamp of the image -/// it was published on. -struct Kept { - name: String, - document: String, - image: String, - base: String, -} - -impl Kept { - fn named(name: String) -> Option { - let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned); - let (document, image, base) = (parts.next()?, parts.next()?, parts.next()?); - Some(Self { - name, - document, - image, - base, - }) - } -} - -/// How long reads may lag this device's own upload before a document still standing as the -/// image the upload was made on counts as having lost it. -const LAG: Duration = Duration::from_secs(60); - -/// A section a document provider keeps, as its replica publishes to it. Each image it -/// publishes is kept, and listed as a version once the document is seen otherwise, until the -/// document holds it: an upload another device's replaced then merges back as a conflict -/// version does (`resolve.rs`). Only this device's publications are kept so. +/// A section a document provider keeps, as its replica publishes to it, each image it +/// publishes kept until the document holds it (`published.rs`). pub(crate) struct ProviderRemote { documents: Arc, path: String, + published: Published, /// The image the step read whole for a stamp, which its read takes. read: Option>, } @@ -412,74 +363,19 @@ impl ProviderRemote { Self { documents: Arc::clone(documents), path: path.to_owned(), + published: Published::new(documents.published(), Some(documents.device.clone())), read: None, } } - fn kept(&self) -> Vec<(Kept, std::time::SystemTime)> { - let entries = crate::fs::read_dir(self.documents.published()) - .into_iter() - .flatten(); - entries - .filter_map(|entry| { - let entry = entry.ok()?; - let kept = Kept::named(entry.file_name().into_string().ok()?)?; - Some((kept, entry.metadata().ok()?.modified().ok()?)) - }) - .collect() - } - - /// Keeps the image `transaction` published, in place of the one it was published on: - /// that kept image with the transaction applied where this device kept it, else read back. - fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { - let folder = self.documents.published(); - let base = digest(transaction.base()); - let document = file_of(&transaction.base().header).ok_or(io::ErrorKind::InvalidData)?; - let on = (self.kept().into_iter()) - .find(|(kept, _)| kept.document == hex(&document) && kept.image == base); - let image = match (written, &on) { - (Written::Image(image), _) => image.to_vec(), - (Written::InPlace(_), Some((kept, _))) => { - let mut image = crate::fs::read(folder.join(&kept.name))?; - transaction - .apply(&mut image) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - image - } - (Written::InPlace(file), None) => { - file.seek(SeekFrom::Start(0))?; - whole(file, crate::MAX_FILE_BYTES)? - } - }; - let name = format!( - "{}-{}-{base}.one", - hex(&document), - digest(&transaction.stamp()) - ); - crate::fs::create_dir_all(&folder)?; - let mut kept = tempfile::NamedTempFile::new_in(&folder)?; - kept.write_all(&image)?; - kept.as_file().sync_all()?; - kept.persist(folder.join(name)) - .map_err(|error| error.error)?; - if let Some((kept, _)) = on { - crate::fs::remove_file(folder.join(kept.name))?; - } - Ok(()) - } - /// This device's own upload cut off, as a provider that writes non-atomically leaves it, /// put back and read again: the newest image kept that the torn bytes begin. Any other /// document that does not parse is left as it is. fn repair(&self, torn: &[u8]) -> io::Result> { - let mut newest = None; - for (kept, at) in self.kept() { - let image = crate::fs::read(self.documents.published().join(&kept.name))?; - if image.len() > torn.len() && image.starts_with(torn) { - newest = newest.max(Some((at, image))); - } - } - let (_, image) = newest.ok_or(io::ErrorKind::WouldBlock)?; + let image = self + .published + .torn(torn)? + .ok_or(io::ErrorKind::WouldBlock)?; let id = self.documents.id(&self.path)?; self.documents.writable(&id)?; self.documents @@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote { self.documents.write( &self.path, |io| transaction.commit(io), - |written| self.keep(written, transaction), + |written| self.published.keep(written, transaction), ) } @@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote { .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(())) } - /// The images kept of this document other than the one it stands as; one whose upload - /// the document still stands before is listed only once reads have had `LAG` to catch up. - /// A document read torn is repaired first, once a step. + /// The images kept of this document other than the one it stands as + /// (`Published::versions`). A document read torn is repaired first, once a step. fn versions(&mut self) -> io::Result> { let (mut stamp, mut image) = self.documents.stamp(&self.path)?; if let Some(torn) = image.as_ref().filter(|image| !parses(image)) @@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote { (stamp, image) = (repaired_stamp, Some(repaired)); } self.read = image; - // A header mid-write names no document; the step's read repairs it. - let Some(document) = file_of(&stamp.header) else { - return Ok(Vec::new()); - }; - let (document, current) = (hex(&document), digest(&stamp)); - let lagging = |kept: &Kept, at: std::time::SystemTime| { - kept.base == current && at.elapsed().is_ok_and(|elapsed| elapsed < LAG) - }; - Ok((self.kept().into_iter()) - .filter(|(kept, at)| { - kept.document == document && kept.image != current && !lagging(kept, *at) - }) - .map(|(kept, _)| crate::Version { - id: kept.name, - device: Some(self.documents.device.clone()), - }) - .collect()) + Ok(self.published.versions(&stamp)) } fn version(&mut self, id: &str) -> io::Result> { - crate::fs::read(self.documents.published().join(id)) + crate::fs::read(id) } /// One that cannot merge, of another section under the document's name, is set aside /// whole, where `versions` lists it no more: no provider here takes a new document beside it. fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { - let path = self.documents.published().join(id); + let path = PathBuf::from(id); if keep { crate::fs::rename(&path, path.with_extension("unmerged")) } else { diff --git a/crates/notebook/src/provider/tests.rs b/crates/notebook/src/provider/tests.rs index 00aebee6d2472fa00e79885fe6093654e7e34204..58fff58ad285f9c53b4e566c996c38d7a906885f 100644 --- a/crates/notebook/src/provider/tests.rs +++ b/crates/notebook/src/provider/tests.rs @@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec { /// The images a device keeps that Notes.one does not stand as, still to merge. fn unmerged(fake: &Fake) -> Vec { - let current = digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap()); + let current = crate::published::digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap()); let mut images = kept_images(fake); - images.retain(|name| Kept::named(name.clone()).is_some_and(|kept| kept.image != current)); + images.retain(|name| name.split('-').nth(1) != Some(current.as_str())); images } @@ -976,7 +976,8 @@ fn aged(fake: &Fake) { .write(true) .open(image.unwrap().path()) .unwrap(); - let past = std::time::SystemTime::now() - LAG - Duration::from_secs(1); + let past = + std::time::SystemTime::now() - crate::published::LAG - Duration::from_secs(1); file.set_modified(past).unwrap(); } } diff --git a/crates/notebook/src/published.rs b/crates/notebook/src/published.rs new file mode 100644 index 0000000000000000000000000000000000000000..a508b289225911fadf6b44c88a919e894242b399 --- /dev/null +++ b/crates/notebook/src/published.rs @@ -0,0 +1,164 @@ +//! The images this device publishes to section files a provider syncs whole, with no lock or +//! compare-and-swap, as iCloud Drive and Android's document providers do: another device's +//! upload can replace this one's. Each image is kept until the file is seen holding it, and +//! one the file lost is listed as a `Remote::versions` version, so that it merges back as a +//! conflict version does (`resolve.rs`). + +use crate::Version; +use onestore::{Stamp, Transaction}; +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + time::{Duration, SystemTime}, +}; + +/// How long reads may lag this device's own upload before a file still standing as the image +/// the upload was made on counts as having lost it. +pub(crate) const LAG: Duration = Duration::from_secs(60); + +/// What a publication wrote: the file in place, or its whole image. +pub enum Written<'a> { + InPlace(&'a mut File), + Image(&'a [u8]), +} + +/// The images this device published to the files of one place, kept in `folder` and named +/// `--.one`: the file's identity, then the digests of the image's stamp +/// and of the stamp of the image it was published on. Version ids are their paths. +pub struct Published { + folder: PathBuf, + /// This device, as the conflict pages a lost publication makes name it. + device: Option, +} + +struct Kept { + path: PathBuf, + document: String, + image: String, + base: String, + at: SystemTime, +} + +impl Published { + pub fn new(folder: impl Into, device: Option) -> Self { + Self { + folder: folder.into(), + device, + } + } + + fn kept(&self) -> Vec { + let entries = crate::fs::read_dir(&self.folder).into_iter().flatten(); + entries + .filter_map(|entry| { + let entry = entry.ok()?; + let name = entry.file_name().into_string().ok()?; + let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned); + Some(Kept { + path: entry.path(), + document: parts.next()?, + image: parts.next()?, + base: parts.next()?, + at: entry.metadata().ok()?.modified().ok()?, + }) + }) + .collect() + } + + /// Keeps the image `transaction` published, in place of the one it was published on: + /// that kept image with the transaction applied where this device kept it, else read back. + pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { + let base = digest(transaction.base()); + let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?; + let on = + (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base); + let image = match (written, &on) { + (Written::Image(image), _) => image.to_vec(), + (Written::InPlace(_), Some(kept)) => { + let mut image = crate::fs::read(&kept.path)?; + transaction + .apply(&mut image) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + image + } + (Written::InPlace(file), None) => { + file.seek(SeekFrom::Start(0))?; + let mut image = Vec::new(); + file.take(crate::MAX_FILE_BYTES as u64 + 1) + .read_to_end(&mut image)?; + if image.len() > crate::MAX_FILE_BYTES { + return Err(io::ErrorKind::FileTooLarge.into()); + } + image + } + }; + let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp())); + crate::fs::create_dir_all(&self.folder)?; + let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?; + kept.write_all(&image)?; + kept.as_file().sync_all()?; + kept.persist(self.folder.join(name)) + .map_err(|error| error.error)?; + if let Some(kept) = on { + crate::fs::remove_file(kept.path)?; + } + Ok(()) + } + + /// The newest image kept that `torn` begins, as a provider writing in place leaves this + /// device's upload cut off. + pub fn torn(&self, torn: &[u8]) -> io::Result>> { + let mut newest = None; + for kept in self.kept() { + let image = crate::fs::read(&kept.path)?; + if image.len() > torn.len() && image.starts_with(torn) { + newest = newest.max(Some((kept.at, image))); + } + } + Ok(newest.map(|(_, image)| image)) + } + + /// The images kept of the file `stamp` stamps, other than the one it stands as; one whose + /// upload the file still stands before is listed only once reads have had `LAG` to catch up. + pub fn versions(&self, stamp: &Stamp) -> Vec { + // A header mid-write names no file. + let Some(document) = document(stamp) else { + return Vec::new(); + }; + let current = digest(stamp); + let lagging = |kept: &Kept| { + kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG) + }; + (self.kept().into_iter()) + .filter(|kept| kept.document == document && kept.image != current && !lagging(kept)) + .map(|kept| Version { + id: kept.path.to_string_lossy().into_owned(), + device: self.device.clone(), + }) + .collect() + } + + /// Whether version `id` is an image kept here. + pub fn holds(&self, id: &str) -> bool { + Path::new(id).parent() == Some(self.folder.as_path()) + } +} + +/// The hex of a stamp's digest, which names an image this device keeps. +pub(crate) fn digest(stamp: &Stamp) -> String { + use sha2::Digest; + let digest = sha2::Sha256::new() + .chain_update(stamp.header) + .chain_update(stamp.length.to_le_bytes()) + .finalize(); + hex(&digest[..16]) +} + +fn document(stamp: &Stamp) -> Option { + Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id)) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +}