authoravatarclover caruso<git@paperclover.net>2026-10-19 11:44:38-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-19 23:39:58-07:00
log02876d691c955ff0594f528e889ff7e7a057fdd9
treea1dd96b38792a7806ccc90ee643d78bbbca98287
parent82d8d2ab063d1f2aa562cb75c3beedea691e2c15
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

fix(ios): merge back uploads iCloud Drive lost

iCloud Drive syncs whole files with no lock, so another device's upload can land over the phone's with no conflict version kept, and the phone's next sync took the file as it stood: its published edits were gone. Android's document providers already kept each published image until the document held it (c345a48e); that store moves into notebook::published, and the iOS file remote uses it for notebooks a provider keeps elsewhere. Each image the phone publishes is kept in the cache until the file is seen otherwise, then listed beside the host's conflict versions: one the file holds merges as nothing and goes, one it lost merges back with conflict pages named for this device, and one that cannot merge is kept beside the file as "Name (Device).one", as a conflict version is. sb_set_versions takes the device's name for these. The iCloud launch test gains the lost upload, with and without shared history; it fails without the store. - Edits you make on iPhone to an iCloud Drive notebook come back if iCloud lets another device's upload replace them, instead of being lost. Assisted-by: claude-opus-5.5

11 files changed, 375 insertions(+), 203 deletions(-)

apps/ios/Snowbound/ICloud.swift+5-4
......@@ -1,4 +1,4 @@
1import Foundation
1import UIKit
22
33/// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another
44/// device's commit that lost; the library merges each into the file.
......@@ -58,10 +58,11 @@ enum ICloud {
5858 /// Whether iCloud is signed in with iCloud Drive on, container or not.
5959 static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil }
6060
61 /// Has the library merge conflict versions, follows the account, and hears of notebooks
62 /// another device adds or removes; call once at launch.
61 /// Has the library merge conflict versions and this device's uploads iCloud Drive lost,
62 /// follows the account, and hears of notebooks another device adds or removes; call once
63 /// at launch.
6364 static func start() {
64 sb_set_versions(listVersions, retireVersion)
65 sb_set_versions(listVersions, retireVersion, UIDevice.current.name)
6566 if automated { return lookUp() }
6667 NotificationCenter.default.addObserver(
6768 forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main
arc/sync.md+8
......@@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai
295295device that wrote it. A version of another section, or one that cannot be read, is kept beside
296296the file as `Name (Device).one` instead.
297297
298iCloud Drive can also lose an upload outright: another device's lands over it and no conflict
299version reaches this one. So on the iPhone and on Android each image a section publishes to a
300file a provider keeps stays in the cache (`notebook::published`) until the file is seen
301otherwise, then lists as one more version: one the file holds merges as nothing and goes, one
302it lost merges back as a conflict version does, its conflict pages named for this device. One
303that cannot merge is kept beside the file as `Name (Device).one` on the iPhone, and in the
304cache on Android, where no provider takes a new document beside it.
305
298306On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the
299307iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted
300308(a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists
crates/mobile/include/snowbound.h+1-1
......@@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator);
2222void sb_set_sync_wake(void (*wake)(void));
2323typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context);
2424typedef bool (*sb_retire)(const char *path, const char *id, bool keep);
25void sb_set_versions(sb_versions versions, sb_retire retire);
25void sb_set_versions(sb_versions versions, sb_retire retire, const char *device);
2626char *sb_tags(void);
2727char *sb_pens(uint32_t section);
2828bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba);
crates/mobile/src/library.rs+117-38
......@@ -9,6 +9,7 @@ use notebook::{
99 discover::{Folder, Reason, SectionState},
1010 live::{self, Hello, Reach, share::Guest},
1111 provider::Documents,
12 published::{Published, Written},
1213 session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus},
1314 smb::{Client, Credentials},
1415};
......@@ -103,28 +104,60 @@ pub type Versions = extern "C" fn(
103104/// beside the file when `keep`; false when it could not.
104105pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool;
105106
106static VERSIONS: OnceLock<(Versions, Retire)> = OnceLock::new();
107static VERSIONS: OnceLock<(Versions, Retire, String)> = OnceLock::new();
107108
108109/// Merges the conflict versions the host lists into their files from now on, then has the
109/// host retire them; set once, before any library opens.
110/// host retire them, and merges back this device's publications a file provider lost, which
111/// conflict pages name for `device`; set once, before any library opens.
112///
113/// # Safety
114/// `device` is NUL-terminated UTF-8.
110115#[unsafe(no_mangle)]
111pub extern "C" fn sb_set_versions(versions: Versions, retire: Retire) {
112 let _ = VERSIONS.set((versions, retire));
116pub unsafe extern "C" fn sb_set_versions(
117 versions: Versions,
118 retire: Retire,
119 device: *const c_char,
120) {
121 let _ = VERSIONS.set((versions, retire, string(device)));
122}
123
124/// A local section file read and published under the host's file coordination. One a file
125/// provider keeps elsewhere, which syncs it whole with no lock, keeps each image this device
126/// publishes until the file holds it, and merges one another device's upload replaced back
127/// (`notebook::published`).
128struct Coordinated {
129 file: PathBuf,
130 published: Option<Published>,
113131}
114132
115/// A local section file read and published under the host's file coordination.
116struct Coordinated(PathBuf);
133impl Coordinated {
134 fn new(file: &Path, cache: &Path, elsewhere: bool) -> Self {
135 let published = elsewhere
136 .then(|| {
137 notebook::location::claimed(cache, &notebook::location::local(file).ok()?).ok()
138 })
139 .flatten()
140 .map(|folder| {
141 let device = VERSIONS.get().map(|(.., device)| device.clone());
142 Published::new(folder.join("published"), device)
143 });
144 Self {
145 file: file.to_owned(),
146 published,
147 }
148 }
149}
117150
118151impl Remote for Coordinated {
119152 fn read(&mut self) -> io::Result<Vec<u8>> {
120 coordinated(&self.0, false, || onestore::read_file(&self.0))?
153 coordinated(&self.file, false, || onestore::read_file(&self.file))?
121154 }
122155
123156 /// The header and length, as the file remote's uncoordinated change detector reads them.
124157 fn stamp(&mut self) -> io::Result<Stamp> {
125 coordinated(&self.0, false, || {
158 coordinated(&self.file, false, || {
126159 use std::io::Read;
127 let mut file = std::fs::File::open(&self.0)?;
160 let mut file = std::fs::File::open(&self.file)?;
128161 let mut header = [0; 1024];
129162 file.read_exact(&mut header)?;
130163 Ok(Stamp {
......@@ -135,28 +168,48 @@ impl Remote for Coordinated {
135168 }
136169
137170 fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> {
138 coordinated(&self.0, true, || transaction.commit_file(&self.0)).map_err(uncommitted)?
171 coordinated(&self.file, true, || {
172 transaction.commit_file(&self.file)?;
173 if let Some(published) = &self.published {
174 // A copy it could not keep only leaves the publication unprotected.
175 let _ = std::fs::File::open(&self.file)
176 .and_then(|mut file| published.keep(Written::InPlace(&mut file), transaction));
177 }
178 Ok(())
179 })
180 .map_err(uncommitted)?
139181 }
140182
141183 fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> {
142 coordinated(&self.0, true, || onestore::confirm_file(&self.0, base)).map_err(uncommitted)?
184 coordinated(&self.file, true, || {
185 onestore::confirm_file(&self.file, base)
186 })
187 .map_err(uncommitted)?
143188 }
144189
190 /// The host's conflict versions, then this device's publications the file lost.
145191 fn versions(&mut self) -> io::Result<Vec<notebook::Version>> {
146 let Some((versions, _)) = VERSIONS.get() else {
147 return Ok(Vec::new());
148 };
149 extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) {
150 // SAFETY: `context` is the list below, alive for the host's call.
151 let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() };
152 listed.push(notebook::Version {
153 id: string(id),
154 device: (!device.is_null()).then(|| string(device)),
155 });
156 }
157192 let mut listed: Vec<notebook::Version> = Vec::new();
158 let path = CString::new(self.0.as_os_str().as_encoded_bytes())?;
159 versions(path.as_ptr(), found, (&raw mut listed).cast());
193 if let Some((versions, ..)) = VERSIONS.get() {
194 extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) {
195 // SAFETY: `context` is the list below, alive for the host's call.
196 let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() };
197 listed.push(notebook::Version {
198 id: string(id),
199 device: (!device.is_null()).then(|| string(device)),
200 });
201 }
202 let path = CString::new(self.file.as_os_str().as_encoded_bytes())?;
203 versions(path.as_ptr(), found, (&raw mut listed).cast());
204 }
205 if self.published.is_some() {
206 let stamp = self.stamp()?;
207 listed.extend(
208 self.published
209 .iter()
210 .flat_map(|published| published.versions(&stamp)),
211 );
212 }
160213 Ok(listed)
161214 }
162215
......@@ -165,11 +218,31 @@ impl Remote for Coordinated {
165218 std::fs::read(id)
166219 }
167220
221 /// A publication of this device's that cannot merge is kept whole beside the file, as
222 /// the host keeps a conflict version: "Name (Device).one".
168223 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {
169 let Some((_, retire)) = VERSIONS.get() else {
224 if let Some(published) = &self.published
225 && published.holds(id)
226 {
227 if keep {
228 let stem = self.file.file_stem().unwrap_or_default().to_string_lossy();
229 let device = VERSIONS.get().map_or("this device", |(.., device)| device);
230 let beside = (1..)
231 .map(|n| match n {
232 1 => format!("{stem} ({device}).one"),
233 _ => format!("{stem} ({device} {n}).one"),
234 })
235 .map(|name| self.file.with_file_name(name))
236 .find(|beside| std::fs::symlink_metadata(beside).is_err())
237 .ok_or(io::ErrorKind::AlreadyExists)?;
238 coordinated(&beside, true, || std::fs::copy(id, &beside))??;
239 }
240 return std::fs::remove_file(id);
241 }
242 let Some((_, retire, _)) = VERSIONS.get() else {
170243 return Ok(());
171244 };
172 let path = CString::new(self.0.as_os_str().as_encoded_bytes())?;
245 let path = CString::new(self.file.as_os_str().as_encoded_bytes())?;
173246 let id = CString::new(id)?;
174247 if retire(path.as_ptr(), id.as_ptr(), keep) {
175248 Ok(())
......@@ -297,10 +370,10 @@ pub struct Library {
297370 offline: AtomicBool,
298371 /// Keys of the protected sections unlocked this run, by catalog path; in memory only.
299372 keys: Mutex<HashMap<String, Key>>,
300 /// How long edits wait for a pause in typing before they publish (`Section::set_pause`):
301 /// a file provider such as iCloud Drive uploads each publication, and makes one concurrent
302 /// with another device's a conflict version.
303 pause: Duration,
373 /// A file provider keeps the notebook elsewhere too, as iCloud Drive does: edits wait for a
374 /// pause in typing before they publish (`Section::set_pause`), as it uploads each
375 /// publication and makes one concurrent with another device's a conflict version.
376 elsewhere: bool,
304377}
305378
306379/// The pause edits to a file a provider keeps elsewhere wait for.
......@@ -400,10 +473,11 @@ impl Library {
400473 (None, Place::File(path.to_owned()), None)
401474 } else {
402475 let mut notebook = Notebook::open(path, cache)?;
476 let kept = cache.to_owned();
403477 let background = notebook.background_with(
404478 local,
405479 !local,
406 |file| Coordinated(file.to_owned()),
480 move |file| Coordinated::new(file, &kept, !local),
407481 sync_woken,
408482 )?;
409483 background.watch(notebook.replicas());
......@@ -422,7 +496,7 @@ impl Library {
422496 open: Mutex::default(),
423497 offline: AtomicBool::new(false),
424498 keys: Default::default(),
425 pause: if local { Duration::ZERO } else { PAUSE },
499 elsewhere: !local,
426500 })
427501 }
428502
......@@ -445,7 +519,7 @@ impl Library {
445519 open: Mutex::default(),
446520 offline: AtomicBool::new(false),
447521 keys: Default::default(),
448 pause: Duration::ZERO,
522 elsewhere: false,
449523 };
450524 let reached = library.with_notebook(false, |_| Ok(()));
451525 match library.listed() {
......@@ -504,7 +578,7 @@ impl Library {
504578 open: Mutex::default(),
505579 offline: AtomicBool::new(offline),
506580 keys: Default::default(),
507 pause: Duration::ZERO,
581 elsewhere: false,
508582 })
509583 }
510584
......@@ -541,7 +615,7 @@ impl Library {
541615 open: Mutex::default(),
542616 offline: AtomicBool::new(false),
543617 keys: Default::default(),
544 pause: if local { Duration::ZERO } else { PAUSE },
618 elsewhere: !local,
545619 })
546620 }
547621
......@@ -783,7 +857,11 @@ impl Library {
783857 }
784858 };
785859 section.set_offline(self.offline.load(Ordering::Relaxed));
786 section.set_pause(self.pause);
860 section.set_pause(if self.elsewhere {
861 PAUSE
862 } else {
863 Duration::ZERO
864 });
787865 if let Some(background) = &self.background {
788866 background.hold(path, &section);
789867 }
......@@ -796,18 +874,19 @@ impl Library {
796874 notify: impl Fn() + Send + Sync + 'static,
797875 ) -> Result<session::Section> {
798876 let offline = self.offline.load(Ordering::Relaxed);
877 let (cache, elsewhere) = (self.cache.clone(), self.elsewhere);
878 let connect = move |file: &Path| Ok(Coordinated::new(file, &cache, elsewhere));
799879 match &self.place {
800880 Place::File(file) => Ok(session::Section::open_with(
801881 file,
802882 &self.cache,
803 |file| Ok(Coordinated(file.to_owned())),
883 connect,
804884 offline,
805885 notify,
806886 )?),
807887 Place::Folder(_) => {
808888 let notebook = self.notebook();
809889 let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?;
810 let connect = |file: &Path| Ok(Coordinated(file.to_owned()));
811890 Ok(match self.unlocked().get(path) {
812891 Some(key) => {
813892 notebook.section_unlocked_with(path, key, connect, offline, notify)?
crates/mobile/src/tests.rs+50-16
......@@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String {
46554655 shown
46564656}
46574657
4658/// The titles and text of every page and conflict page each section file in `root` stores.
4659fn stored(root: &Path) -> String {
4660 let mut text = String::new();
4661 for entry in std::fs::read_dir(root).unwrap() {
4662 let path = entry.unwrap().path();
4663 if path.extension().is_some_and(|extension| extension == "one") {
4664 let arena = onestore::Arena::default();
4665 let mut section =
4666 onestore::Section::open(&arena, std::fs::read(path).unwrap()).unwrap();
4667 let conflicts = section.conflicts().unwrap().into_iter();
4668 let spaces = (section
4669 .pages()
4670 .unwrap()
4671 .into_iter()
4672 .map(|(space, ..)| space))
4673 .chain(
4674 conflicts
4675 .flat_map(|(_, pages)| pages)
4676 .map(|page| page.space),
4677 );
4678 for space in spaces.collect::<Vec<_>>() {
4679 let page = section.page(space).unwrap();
4680 text += &format!("{}\n{}\n", page.title, canvas::search::page_text(&page));
4681 }
4682 }
4683 }
4684 text
4685}
4686
46584687/// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy:
46594688/// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone
4660/// types on it, and a section the phone adds. The Mac's upload wins, so the phone's next
4661/// launch finds its own file a conflict version, and opens, lists and merges it through the C
4662/// surface as iOS does, its pages open and typed on in the Reading View all the while.
4663/// Nothing either device typed goes missing, merged or set aside whole.
4689/// types on it, and a section the phone adds. The Mac's upload wins: iCloud Drive keeps the
4690/// phone's file as a conflict version, or loses it. The phone's next launch opens, lists and
4691/// merges it back through the C surface as iOS does, its pages open and typed on in the
4692/// Reading View all the while. Nothing either device typed goes missing, merged or set aside
4693/// whole.
46644694#[test]
4665fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {
4666 library::sb_set_versions(cloud_versions, cloud_retire);
4667 // Whether the devices' copies already share an edit of the page both type on.
4668 for shared in [false, true] {
4695fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() {
4696 unsafe {
4697 library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr())
4698 };
4699 // Whether the devices' copies already share an edit of the page both type on, and
4700 // whether iCloud Drive lost the phone's upload rather than keep it as a version.
4701 for (shared, lost) in [(false, false), (true, false), (false, true), (true, true)] {
46694702 let directory = tempfile::tempdir().unwrap();
46704703 let folder = |name: &str| directory.path().join(name);
46714704 let (phone, mac) = (folder("phone"), folder("mac"));
......@@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {
47394772 let version = folder("version.one");
47404773 std::fs::rename(&file, &version).unwrap();
47414774 std::fs::copy(mac.join(&path), &file).unwrap();
4742 let device = "Clover’s iPhone".to_owned();
4743 CLOUD_VERSIONS.lock().unwrap().push((
4744 std::fs::canonicalize(&file).unwrap(),
4745 version,
4746 device,
4747 ));
4775 if !lost {
4776 let device = "Clover’s iPhone".to_owned();
4777 let file = std::fs::canonicalize(&file).unwrap();
4778 CLOUD_VERSIONS.lock().unwrap().push((file, version, device));
4779 }
47484780
47494781 // The launch: the notebook opens and lists, the page shown last opens, and the
47504782 // background merges the version meanwhile.
......@@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {
47614793 .collect();
47624794 sb_library_sync_now(&library);
47634795 let deadline = Instant::now() + Duration::from_secs(10);
4764 while !CLOUD_VERSIONS.lock().unwrap().is_empty() {
4765 assert!(Instant::now() < deadline, "the version never merged");
4796 while !CLOUD_VERSIONS.lock().unwrap().is_empty()
4797 || !stored(&phone).contains("Phone first words")
4798 {
4799 assert!(Instant::now() < deadline, "the phone's upload never merged");
47664800 library::sb_section_poll(&section, &mut 0);
47674801 for view in &mut views {
47684802 unsafe { sb_insert(view, c"Launch ".as_ptr()) };
crates/notebook/README.md+5
......@@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg
151151(`Section::seal_as`), so a version merged once, here or on another device, merges as nothing
152152again. It is published on the file's stamp, then retired; one of another section, or one that
153153cannot be read, is retired with `keep` for the remote to keep beside the file.
154A provider can also lose an upload outright, another device's landing over it with no version
155kept. A remote keeps what it publishes with `published::Published` (`keep` after each
156publication) and lists each image the file no longer stands as among its versions
157(`Published::versions`): one the file holds merges as nothing and is dropped, one it lost
158merges back.
154159
155160Publication attempts are recorded before network I/O. An attempt confirms only
156161when the remote holds its revisions, or every page it changed as it changed them,
crates/notebook/src/lib.rs+1
......@@ -33,6 +33,7 @@ mod merge;
3333mod migrate;
3434pub mod package;
3535pub mod provider;
36pub mod published;
3637mod queue;
3738mod recovery;
3839mod resolve;
crates/notebook/src/location.rs+1-1
......@@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> {
8484}
8585
8686/// The folder for `location`, made and labelled with it where it was not.
87fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> {
87pub fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> {
8888 let folder = folder(cache, location);
8989 let label = folder.join(LOCATION);
9090 if fs::metadata(&label).is_err() {
crates/notebook/src/provider.rs+19-140
......@@ -4,7 +4,11 @@
44//! notebook root (`session::Notebook::open_provider`), a picked document a lone section
55//! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache.
66
7use crate::{Error, Result, discover, session::Storage};
7use crate::{
8 Error, Result, discover,
9 published::{Published, Written},
10 session::Storage,
11};
812use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction};
913use std::{
1014 collections::BTreeMap,
......@@ -12,7 +16,6 @@ use std::{
1216 io::{self, Read, Seek, SeekFrom, Write},
1317 path::PathBuf,
1418 sync::{Arc, Mutex, MutexGuard, PoisonError},
15 time::Duration,
1619};
1720
1821/// A document as its provider describes it.
......@@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> {
294297 Ok(bytes)
295298}
296299
297/// What a write wrote: the document in place, or its whole image.
298enum Written<'a> {
299 InPlace(&'a mut File),
300 Image(&'a [u8]),
301}
302
303300/// A document as a commit writes it: its seekable descriptor, or its image to write whole.
304301enum Io {
305302 File(File),
......@@ -351,58 +348,12 @@ impl CommitIo for Io {
351348 }
352349}
353350
354/// The hex of a stamp's digest, which names an image this device keeps.
355fn digest(stamp: &Stamp) -> String {
356 use sha2::Digest;
357 let digest = sha2::Sha256::new()
358 .chain_update(stamp.header)
359 .chain_update(stamp.length.to_le_bytes())
360 .finalize();
361 hex(&digest[..16])
362}
363
364fn hex(bytes: &[u8]) -> String {
365 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
366}
367
368fn file_of(header: &[u8]) -> Option<[u8; 16]> {
369 Some(onestore::Header::parse(header).ok()?.file_id)
370}
371
372/// An image this device keeps, as its name gives it: `<document>-<image>-<base>.one`, the
373/// document's identity, then the digests of the image's stamp and of the stamp of the image
374/// it was published on.
375struct Kept {
376 name: String,
377 document: String,
378 image: String,
379 base: String,
380}
381
382impl Kept {
383 fn named(name: String) -> Option<Self> {
384 let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned);
385 let (document, image, base) = (parts.next()?, parts.next()?, parts.next()?);
386 Some(Self {
387 name,
388 document,
389 image,
390 base,
391 })
392 }
393}
394
395/// How long reads may lag this device's own upload before a document still standing as the
396/// image the upload was made on counts as having lost it.
397const LAG: Duration = Duration::from_secs(60);
398
399/// A section a document provider keeps, as its replica publishes to it. Each image it
400/// publishes is kept, and listed as a version once the document is seen otherwise, until the
401/// document holds it: an upload another device's replaced then merges back as a conflict
402/// version does (`resolve.rs`). Only this device's publications are kept so.
351/// A section a document provider keeps, as its replica publishes to it, each image it
352/// publishes kept until the document holds it (`published.rs`).
403353pub(crate) struct ProviderRemote {
404354 documents: Arc<Documents>,
405355 path: String,
356 published: Published,
406357 /// The image the step read whole for a stamp, which its read takes.
407358 read: Option<Vec<u8>>,
408359}
......@@ -412,74 +363,19 @@ impl ProviderRemote {
412363 Self {
413364 documents: Arc::clone(documents),
414365 path: path.to_owned(),
366 published: Published::new(documents.published(), Some(documents.device.clone())),
415367 read: None,
416368 }
417369 }
418370
419 fn kept(&self) -> Vec<(Kept, std::time::SystemTime)> {
420 let entries = crate::fs::read_dir(self.documents.published())
421 .into_iter()
422 .flatten();
423 entries
424 .filter_map(|entry| {
425 let entry = entry.ok()?;
426 let kept = Kept::named(entry.file_name().into_string().ok()?)?;
427 Some((kept, entry.metadata().ok()?.modified().ok()?))
428 })
429 .collect()
430 }
431
432 /// Keeps the image `transaction` published, in place of the one it was published on:
433 /// that kept image with the transaction applied where this device kept it, else read back.
434 fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> {
435 let folder = self.documents.published();
436 let base = digest(transaction.base());
437 let document = file_of(&transaction.base().header).ok_or(io::ErrorKind::InvalidData)?;
438 let on = (self.kept().into_iter())
439 .find(|(kept, _)| kept.document == hex(&document) && kept.image == base);
440 let image = match (written, &on) {
441 (Written::Image(image), _) => image.to_vec(),
442 (Written::InPlace(_), Some((kept, _))) => {
443 let mut image = crate::fs::read(folder.join(&kept.name))?;
444 transaction
445 .apply(&mut image)
446 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
447 image
448 }
449 (Written::InPlace(file), None) => {
450 file.seek(SeekFrom::Start(0))?;
451 whole(file, crate::MAX_FILE_BYTES)?
452 }
453 };
454 let name = format!(
455 "{}-{}-{base}.one",
456 hex(&document),
457 digest(&transaction.stamp())
458 );
459 crate::fs::create_dir_all(&folder)?;
460 let mut kept = tempfile::NamedTempFile::new_in(&folder)?;
461 kept.write_all(&image)?;
462 kept.as_file().sync_all()?;
463 kept.persist(folder.join(name))
464 .map_err(|error| error.error)?;
465 if let Some((kept, _)) = on {
466 crate::fs::remove_file(folder.join(kept.name))?;
467 }
468 Ok(())
469 }
470
471371 /// This device's own upload cut off, as a provider that writes non-atomically leaves it,
472372 /// put back and read again: the newest image kept that the torn bytes begin. Any other
473373 /// document that does not parse is left as it is.
474374 fn repair(&self, torn: &[u8]) -> io::Result<Vec<u8>> {
475 let mut newest = None;
476 for (kept, at) in self.kept() {
477 let image = crate::fs::read(self.documents.published().join(&kept.name))?;
478 if image.len() > torn.len() && image.starts_with(torn) {
479 newest = newest.max(Some((at, image)));
480 }
481 }
482 let (_, image) = newest.ok_or(io::ErrorKind::WouldBlock)?;
375 let image = self
376 .published
377 .torn(torn)?
378 .ok_or(io::ErrorKind::WouldBlock)?;
483379 let id = self.documents.id(&self.path)?;
484380 self.documents.writable(&id)?;
485381 self.documents
......@@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote {
524420 self.documents.write(
525421 &self.path,
526422 |io| transaction.commit(io),
527 |written| self.keep(written, transaction),
423 |written| self.published.keep(written, transaction),
528424 )
529425 }
530426
......@@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote {
534430 .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(()))
535431 }
536432
537 /// The images kept of this document other than the one it stands as; one whose upload
538 /// the document still stands before is listed only once reads have had `LAG` to catch up.
539 /// A document read torn is repaired first, once a step.
433 /// The images kept of this document other than the one it stands as
434 /// (`Published::versions`). A document read torn is repaired first, once a step.
540435 fn versions(&mut self) -> io::Result<Vec<crate::Version>> {
541436 let (mut stamp, mut image) = self.documents.stamp(&self.path)?;
542437 if let Some(torn) = image.as_ref().filter(|image| !parses(image))
......@@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote {
546441 (stamp, image) = (repaired_stamp, Some(repaired));
547442 }
548443 self.read = image;
549 // A header mid-write names no document; the step's read repairs it.
550 let Some(document) = file_of(&stamp.header) else {
551 return Ok(Vec::new());
552 };
553 let (document, current) = (hex(&document), digest(&stamp));
554 let lagging = |kept: &Kept, at: std::time::SystemTime| {
555 kept.base == current && at.elapsed().is_ok_and(|elapsed| elapsed < LAG)
556 };
557 Ok((self.kept().into_iter())
558 .filter(|(kept, at)| {
559 kept.document == document && kept.image != current && !lagging(kept, *at)
560 })
561 .map(|(kept, _)| crate::Version {
562 id: kept.name,
563 device: Some(self.documents.device.clone()),
564 })
565 .collect())
444 Ok(self.published.versions(&stamp))
566445 }
567446
568447 fn version(&mut self, id: &str) -> io::Result<Vec<u8>> {
569 crate::fs::read(self.documents.published().join(id))
448 crate::fs::read(id)
570449 }
571450
572451 /// One that cannot merge, of another section under the document's name, is set aside
573452 /// whole, where `versions` lists it no more: no provider here takes a new document beside it.
574453 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {
575 let path = self.documents.published().join(id);
454 let path = PathBuf::from(id);
576455 if keep {
577456 crate::fs::rename(&path, path.with_extension("unmerged"))
578457 } else {
crates/notebook/src/provider/tests.rs+4-3
......@@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec<String> {
917917
918918/// The images a device keeps that Notes.one does not stand as, still to merge.
919919fn unmerged(fake: &Fake) -> Vec<String> {
920 let current = digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap());
920 let current = crate::published::digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap());
921921 let mut images = kept_images(fake);
922 images.retain(|name| Kept::named(name.clone()).is_some_and(|kept| kept.image != current));
922 images.retain(|name| name.split('-').nth(1) != Some(current.as_str()));
923923 images
924924}
925925
......@@ -976,7 +976,8 @@ fn aged(fake: &Fake) {
976976 .write(true)
977977 .open(image.unwrap().path())
978978 .unwrap();
979 let past = std::time::SystemTime::now() - LAG - Duration::from_secs(1);
979 let past =
980 std::time::SystemTime::now() - crate::published::LAG - Duration::from_secs(1);
980981 file.set_modified(past).unwrap();
981982 }
982983 }
crates/notebook/src/published.rs created+164
......@@ -0,0 +1,164 @@
1//! The images this device publishes to section files a provider syncs whole, with no lock or
2//! compare-and-swap, as iCloud Drive and Android's document providers do: another device's
3//! upload can replace this one's. Each image is kept until the file is seen holding it, and
4//! one the file lost is listed as a `Remote::versions` version, so that it merges back as a
5//! conflict version does (`resolve.rs`).
6
7use crate::Version;
8use onestore::{Stamp, Transaction};
9use std::{
10 fs::File,
11 io::{self, Read, Seek, SeekFrom, Write},
12 path::{Path, PathBuf},
13 time::{Duration, SystemTime},
14};
15
16/// How long reads may lag this device's own upload before a file still standing as the image
17/// the upload was made on counts as having lost it.
18pub(crate) const LAG: Duration = Duration::from_secs(60);
19
20/// What a publication wrote: the file in place, or its whole image.
21pub enum Written<'a> {
22 InPlace(&'a mut File),
23 Image(&'a [u8]),
24}
25
26/// The images this device published to the files of one place, kept in `folder` and named
27/// `<document>-<image>-<base>.one`: the file's identity, then the digests of the image's stamp
28/// and of the stamp of the image it was published on. Version ids are their paths.
29pub struct Published {
30 folder: PathBuf,
31 /// This device, as the conflict pages a lost publication makes name it.
32 device: Option<String>,
33}
34
35struct Kept {
36 path: PathBuf,
37 document: String,
38 image: String,
39 base: String,
40 at: SystemTime,
41}
42
43impl Published {
44 pub fn new(folder: impl Into<PathBuf>, device: Option<String>) -> Self {
45 Self {
46 folder: folder.into(),
47 device,
48 }
49 }
50
51 fn kept(&self) -> Vec<Kept> {
52 let entries = crate::fs::read_dir(&self.folder).into_iter().flatten();
53 entries
54 .filter_map(|entry| {
55 let entry = entry.ok()?;
56 let name = entry.file_name().into_string().ok()?;
57 let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned);
58 Some(Kept {
59 path: entry.path(),
60 document: parts.next()?,
61 image: parts.next()?,
62 base: parts.next()?,
63 at: entry.metadata().ok()?.modified().ok()?,
64 })
65 })
66 .collect()
67 }
68
69 /// Keeps the image `transaction` published, in place of the one it was published on:
70 /// that kept image with the transaction applied where this device kept it, else read back.
71 pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> {
72 let base = digest(transaction.base());
73 let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?;
74 let on =
75 (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base);
76 let image = match (written, &on) {
77 (Written::Image(image), _) => image.to_vec(),
78 (Written::InPlace(_), Some(kept)) => {
79 let mut image = crate::fs::read(&kept.path)?;
80 transaction
81 .apply(&mut image)
82 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
83 image
84 }
85 (Written::InPlace(file), None) => {
86 file.seek(SeekFrom::Start(0))?;
87 let mut image = Vec::new();
88 file.take(crate::MAX_FILE_BYTES as u64 + 1)
89 .read_to_end(&mut image)?;
90 if image.len() > crate::MAX_FILE_BYTES {
91 return Err(io::ErrorKind::FileTooLarge.into());
92 }
93 image
94 }
95 };
96 let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp()));
97 crate::fs::create_dir_all(&self.folder)?;
98 let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?;
99 kept.write_all(&image)?;
100 kept.as_file().sync_all()?;
101 kept.persist(self.folder.join(name))
102 .map_err(|error| error.error)?;
103 if let Some(kept) = on {
104 crate::fs::remove_file(kept.path)?;
105 }
106 Ok(())
107 }
108
109 /// The newest image kept that `torn` begins, as a provider writing in place leaves this
110 /// device's upload cut off.
111 pub fn torn(&self, torn: &[u8]) -> io::Result<Option<Vec<u8>>> {
112 let mut newest = None;
113 for kept in self.kept() {
114 let image = crate::fs::read(&kept.path)?;
115 if image.len() > torn.len() && image.starts_with(torn) {
116 newest = newest.max(Some((kept.at, image)));
117 }
118 }
119 Ok(newest.map(|(_, image)| image))
120 }
121
122 /// The images kept of the file `stamp` stamps, other than the one it stands as; one whose
123 /// upload the file still stands before is listed only once reads have had `LAG` to catch up.
124 pub fn versions(&self, stamp: &Stamp) -> Vec<Version> {
125 // A header mid-write names no file.
126 let Some(document) = document(stamp) else {
127 return Vec::new();
128 };
129 let current = digest(stamp);
130 let lagging = |kept: &Kept| {
131 kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG)
132 };
133 (self.kept().into_iter())
134 .filter(|kept| kept.document == document && kept.image != current && !lagging(kept))
135 .map(|kept| Version {
136 id: kept.path.to_string_lossy().into_owned(),
137 device: self.device.clone(),
138 })
139 .collect()
140 }
141
142 /// Whether version `id` is an image kept here.
143 pub fn holds(&self, id: &str) -> bool {
144 Path::new(id).parent() == Some(self.folder.as_path())
145 }
146}
147
148/// The hex of a stamp's digest, which names an image this device keeps.
149pub(crate) fn digest(stamp: &Stamp) -> String {
150 use sha2::Digest;
151 let digest = sha2::Sha256::new()
152 .chain_update(stamp.header)
153 .chain_update(stamp.length.to_le_bytes())
154 .finalize();
155 hex(&digest[..16])
156}
157
158fn document(stamp: &Stamp) -> Option<String> {
159 Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id))
160}
161
162fn hex(bytes: &[u8]) -> String {
163 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
164}