1use super::*;
2use crate::{PendingEdit, merge};
3use onestore::{Arena, ExGuid, Section, op::Edit};
4use std::collections::BTreeSet;
5
6#[derive(serde::Serialize, serde::Deserialize)]
7#[serde(deny_unknown_fields)]
8pub(super) struct Edits {
9 pub edits: Vec<(String, Edit)>,
10 pub revisions: BTreeMap<ExGuid, ExGuid>,
11}
12
13pub(super) fn encode(edits: &Edits, limit: usize) -> io::Result<Vec<u8>> {
14 struct Buffer {
15 bytes: Vec<u8>,
16 limit: usize,
17 }
18 impl io::Write for Buffer {
19 fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
20 if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
21 return Err(io::ErrorKind::FileTooLarge.into());
22 }
23 self.bytes.extend_from_slice(bytes);
24 Ok(bytes.len())
25 }
26
27 fn flush(&mut self) -> io::Result<()> {
28 Ok(())
29 }
30 }
31 let mut buffer = Buffer {
32 bytes: Vec::new(),
33 limit,
34 };
35 serde_json::to_writer(&mut buffer, edits).map_err(|error| {
36 io::Error::new(
37 error.io_error_kind().unwrap_or(io::ErrorKind::InvalidData),
38 error,
39 )
40 })?;
41 Ok(buffer.bytes)
42}
43
44pub(super) struct Waiting {
45 peer: [u8; 16],
46 request: Request,
47 reply: mpsc::Sender<Reply>,
48}
49
50fn retry(message: &str) -> Error {
51 Error::Remote(CommitError {
52 state: CommitState::NotCommitted,
53 error: io::Error::new(io::ErrorKind::ResourceBusy, message.to_owned()),
54 })
55}
56
57impl Served {
58 pub(super) fn batch(self: &Arc<Self>, peer: &[u8; 16], request: Request) -> Result<Reply> {
59 let path = request.path.clone();
60 if !self.writers.lock().unwrap().contains_key(&path) {
61 let image = self.image(&path)?;
62 let arena = Arena::default();
63 Section::open(&arena, (*image).clone())?;
64 }
65 let mut writers = self.writers.lock().unwrap();
66 let writer = writers.entry(path.clone()).or_insert_with(|| {
67 let (send, receive) = mpsc::sync_channel::<Waiting>(64);
68 let served = Arc::downgrade(self);
69 thread::spawn(move || {
70 while let Ok(first) = receive.recv() {
71 let mut waiting = vec![first];
72 if let Ok(next) = receive.recv_timeout(Duration::from_millis(5)) {
73 waiting.push(next);
74 }
75 waiting.extend(receive.try_iter().take(62));
76 let Some(served) = served.upgrade() else {
77 return;
78 };
79 served.write_batches(&path, waiting);
80 }
81 });
82 send
83 });
84 let (reply, receive) = mpsc::channel();
85 writer
86 .try_send(Waiting {
87 peer: *peer,
88 request,
89 reply,
90 })
91 .map_err(|_| retry("The section's writer is busy"))?;
92 drop(writers);
93 receive.recv_timeout(TIMEOUT).map_err(|_| {
94 Error::Remote(CommitError {
95 state: CommitState::Unknown,
96 error: io::Error::new(
97 io::ErrorKind::TimedOut,
98 "The section's writer did not answer",
99 ),
100 })
101 })
102 }
103
104 fn write_batches(&self, path: &str, waiting: Vec<Waiting>) {
105 let before = match self.image(path) {
106 Ok(image) => image,
107 Err(error) => {
108 for batch in waiting {
109 let _ = batch.reply.send(failed(0, retry(&error.to_string())));
110 }
111 return;
112 }
113 };
114 let mut image = Arc::clone(&before);
115 let mut combined: Option<Transaction> = None;
116 let mut accepted = Vec::new();
117 let mut landing = Vec::new();
118 for batch in waiting {
119 match self.prepare_batch(path, &image, &batch) {
120 Ok((Some(transaction), revisions)) => {
121 let mut next = (*image).clone();
122 let extended =
123 transaction
124 .apply(&mut next)
125 .and_then(|()| match &mut combined {
126 Some(combined) => combined.extend(transaction),
127 None => {
128 combined = Some(transaction);
129 Ok(())
130 }
131 });
132 match extended {
133 Ok(()) => {
134 image = Arc::new(next);
135 landing.push((batch.peer, revisions));
136 accepted.push(batch);
137 }
138 Err(error) => {
139 let _ = batch.reply.send(failed(0, error.into()));
140 }
141 }
142 }
143 Ok((None, _)) => accepted.push(batch),
144 Err(error) => {
145 let _ = batch.reply.send(failed(0, error));
146 }
147 }
148 }
149 if accepted.is_empty() {
150 return;
151 }
152 let result = match &combined {
153 Some(transaction) => self.storage.commit(path, transaction),
154 None => self
155 .storage
156 .confirm(path, &Stamp::of(&image).expect("section stamp"))
157 .map_err(Error::from)
158 .and_then(|()| {
159 image = self.image(path).map_err(|error| CommitError {
160 state: CommitState::Committed,
161 error: match error {
162 Error::Io(error) | Error::RemoteIo(error) => error,
163 Error::Remote(error) => error.error,
164 error => io::Error::other(error),
165 },
166 })?;
167 Ok(())
168 }),
169 };
170 if result.is_ok() {
171 self.tell_delta(path, &before, &image, None);
172 self.keep(path, Arc::clone(&image));
173 self.changed(&[path.to_owned()]);
174 }
175 for batch in accepted {
176 let reply = match &result {
177 Ok(()) => Reply {
178 stamp: Some((&Stamp::of(&image).expect("section stamp")).into()),
179 ..Reply::default()
180 },
181 Err(Error::Remote(error)) => failed(
182 0,
183 Error::Remote(CommitError {
184 state: error.state,
185 error: io::Error::new(error.error.kind(), error.error.to_string()),
186 }),
187 ),
188 Err(error) => failed(0, retry(&error.to_string())),
189 };
190 let _ = batch.reply.send(reply);
191 }
192 if result.is_ok() {
193 for (peer, revisions) in landing {
194 self.remember(&peer, path, Landed::Batch(revisions));
195 }
196 }
197 }
198
199 fn prepare_batch(
200 &self,
201 path: &str,
202 image: &[u8],
203 batch: &Waiting,
204 ) -> Result<(Option<Transaction>, BTreeMap<ExGuid, ExGuid>)> {
205 if !self.guests.lock().unwrap().contains_key(&batch.peer) {
206 return Err(refused(
207 io::ErrorKind::PermissionDenied,
208 "The device is no longer connected",
209 ));
210 }
211 let stamp: Stamp = batch
212 .request
213 .stamp
214 .as_ref()
215 .ok_or_else(|| retry("No batch base"))?
216 .try_into()?;
217 let edits: Edits = serde_json::from_slice(&self.carried(&batch.peer, &batch.request)?)
218 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
219 if edits.revisions.is_empty()
220 || edits
221 .revisions
222 .values()
223 .any(|revision| revision.guid == [0; 16])
224 {
225 return Err(retry("No batch revisions"));
226 }
227 let store = Store::parse(image)?;
228 let index = RevisionIndex::parse(&store)?;
229 let landed = match self.landed(&batch.peer, path) {
230 Some(Landed::Batch(landed)) => landed,
231 _ => BTreeMap::new(),
232 };
233 if edits.revisions.iter().all(|(space, revision)| {
234 landed.get(space) == Some(revision)
235 || (index.spaces.get(space))
236 .is_some_and(|space| space.revisions.contains_key(revision))
237 }) {
238 return Ok((None, edits.revisions));
239 }
240 let names = edits
241 .revisions
242 .iter()
243 .filter(|(space, revision)| {
244 !index
245 .spaces
246 .get(space)
247 .is_some_and(|space| space.revisions.contains_key(revision))
248 })
249 .map(|(space, revision)| (*space, *revision))
250 .collect();
251 let arena = Arena::default();
252 let mut next = Section::open(&arena, image.to_vec())?;
253 let queued: Vec<PendingEdit> = edits
254 .edits
255 .into_iter()
256 .enumerate()
257 .map(|(id, (author, edit))| PendingEdit {
258 id: id as u64,
259 author,
260 edit,
261 })
262 .collect();
263 if Stamp::of(image)? == stamp {
264 for edit in &queued {
265 next.apply(&edit.author, &edit.edit)?;
266 }
267 } else {
268 let base = self
269 .images
270 .lock()
271 .unwrap()
272 .iter()
273 .find_map(|(held, at, image)| {
274 (held == path && *at == stamp).then(|| Arc::clone(image))
275 })
276 .ok_or_else(|| {
277 Error::Remote(CommitError {
278 state: CommitState::NotCommitted,
279 error: io::Error::new(
280 io::ErrorKind::StaleNetworkFileHandle,
281 "The batch's base is no longer held",
282 ),
283 })
284 })?;
285 let old_arena = Arena::default();
286 let mut old = Section::open(&old_arena, (*base).clone())?;
287 if old.root() != next.root() {
288 return Err(retry("The batch belongs to another section"));
289 }
290 let merged = merge::rebase(&mut old, &mut next, &queued, &BTreeSet::new())?;
291 if !merged.conflicts.is_empty() {
292 let local_arena = Arena::default();
293 let mut local = Section::open(&local_arena, (*base).clone())?;
294 for edit in &queued {
295 local.apply(&edit.author, &edit.edit)?;
296 }
297 for (space, (author, objects)) in &merged.conflicts {
298 let page = local.page(*space)?;
299 if next.page(*space).is_ok_and(|remote| remote == page) {
300 return Err(retry("The batch's edits already landed"));
301 }
302 merge::conflict_page(
303 &mut next,
304 &mut local,
305 &merged.moved,
306 *space,
307 &page,
308 author,
309 objects,
310 crate::now(),
311 None,
312 )?;
313 }
314 }
315 }
316 let transaction = next.seal_as(&names)?;
317 if edits.revisions.iter().any(|(space, revision)| {
318 !next
319 .newest()
320 .any(|(held, at)| held == *space && at == *revision)
321 }) {
322 return Err(refused(
323 io::ErrorKind::Unsupported,
324 "The batch needs a guest's commit",
325 ));
326 }
327 Ok((transaction, edits.revisions))
328 }
329}
330
331#[cfg(test)]
332mod tests {
333 use super::*;
334 use onestore::{
335 op::{Op, PageOp},
336 page::{Attachment, PageObject},
337 };
338
339 #[cfg(not(target_arch = "wasm32"))]
340 #[test]
341 fn invalid_sections_do_not_retain_writers_and_can_be_repaired() {
342 let directory = tempfile::tempdir().unwrap();
343 let folder = directory.path().join("notebook");
344 std::fs::create_dir(&folder).unwrap();
345 let served = Arc::new(Served {
346 storage: crate::session::Notebook::open(&folder, directory.path().join("cache"))
347 .unwrap()
348 .into_storage(),
349 images: Mutex::default(),
350 snapshots: Mutex::default(),
351 puts: Mutex::default(),
352 guests: Mutex::default(),
353 writers: Mutex::default(),
354 members: Weak::new(),
355 host: Mutex::default(),
356 room: Mutex::default(),
357 });
358 let peer = [1; 16];
359 let (line, _outbound) = mpsc::channel();
360 let (queue, _requests) = mpsc::sync_channel(1);
361 served.guests.lock().unwrap().insert(
362 peer,
363 Admitted {
364 hello: Arc::new(Hello {
365 peer,
366 ..Hello::new("Guest".into(), None).unwrap()
367 }),
368 device: [0; 16],
369 line: crate::live::Line(line),
370 queue,
371 starts: BURST,
372 counted: Instant::now(),
373 held: Vec::new(),
374 },
375 );
376 let toc = onestore::create_table_of_contents("Invalid.onetoc2", &[]).unwrap();
377 {
378 let store = Store::parse(&toc).unwrap();
379 RevisionIndex::parse(&store).unwrap();
380 }
381 let arena = Arena::default();
382 assert!(Section::open(&arena, toc.clone()).is_err());
383 for image in [None, Some(vec![0; 1024]), Some(toc)] {
384 for at in 0..128 {
385 let path = format!("Missing{at}.one");
386 if let Some(image) = &image {
387 std::fs::write(folder.join(&path), image).unwrap();
388 }
389 let reply = served.handle(
390 &peer,
391 kind::EDITS,
392 &minicbor::to_vec(Request {
393 path,
394 ..Request::default()
395 })
396 .unwrap(),
397 );
398 assert!(reply.failure.is_some());
399 }
400 assert!(served.writers.lock().unwrap().is_empty());
401 }
402
403 let path = "Missing0.one";
404 let image = onestore::create_section(path, "Original text", "Fixture").unwrap();
405 std::fs::write(folder.join(path), &image).unwrap();
406 let store = Store::parse(&image).unwrap();
407 let index = RevisionIndex::parse(&store).unwrap();
408 let document = onestore::document::Document::parse(&index).unwrap();
409 let (space, text) = document
410 .spaces
411 .iter()
412 .find_map(|(space, object)| {
413 object.revisions[&object.contexts[&ExGuid::default()]]
414 .nodes
415 .iter()
416 .find_map(|(id, node)| {
417 matches!(node.kind, onestore::document::Kind::RichText { .. })
418 .then_some((*space, *id))
419 })
420 })
421 .unwrap();
422 let edit = Edit {
423 at: 134_000_000_000_000_000,
424 ops: vec![Op::Page {
425 space,
426 op: PageOp::Text {
427 text,
428 range: 13..13,
429 with: "X".into(),
430 },
431 }],
432 };
433 let arena = Arena::default();
434 let mut section = Section::open(&arena, image).unwrap();
435 section.apply("Alice", &edit).unwrap();
436 let transaction = section.seal().unwrap().unwrap();
437 let request = Request {
438 path: path.into(),
439 stamp: Some(transaction.base().into()),
440 bytes: Some(
441 encode(
442 &Edits {
443 edits: vec![("Alice".into(), edit)],
444 revisions: section.newest().filter(|(at, _)| *at == space).collect(),
445 },
446 LIMIT,
447 )
448 .unwrap(),
449 ),
450 ..Request::default()
451 };
452 let first = served.batch(&peer, request.clone()).unwrap();
453 assert!(first.failure.is_none(), "{:?}", first.failure);
454 let committed = std::fs::read(folder.join(path)).unwrap();
455 let stamp = Stamp::of(&committed).unwrap();
456 assert_eq!(
457 Stamp::try_from(first.stamp.as_ref().unwrap()).unwrap(),
458 stamp
459 );
460 let read_arena = Arena::default();
461 let reread = Section::open(&read_arena, committed.clone()).unwrap();
462 assert_eq!(reread.page(space).unwrap(), section.page(space).unwrap());
463 let repeated = served.batch(&peer, request).unwrap();
464 assert!(repeated.failure.is_none(), "{:?}", repeated.failure);
465 let replayed = std::fs::read(folder.join(path)).unwrap();
466 let confirmed = Stamp::of(&replayed).unwrap();
467 assert_eq!(
468 Stamp::try_from(repeated.stamp.as_ref().unwrap()).unwrap(),
469 confirmed
470 );
471 {
472 let images = served.images.lock().unwrap();
473 let (_, held, image) = images.iter().find(|(held, _, _)| held == path).unwrap();
474 assert_eq!(*held, confirmed);
475 assert_eq!(**image, replayed);
476 }
477 assert_eq!(&replayed[1024..], &committed[1024..]);
478 let replay_arena = Arena::default();
479 let replay = Section::open(&replay_arena, replayed).unwrap();
480 assert_eq!(replay.page(space).unwrap(), section.page(space).unwrap());
481 assert_eq!(served.writers.lock().unwrap().len(), 1);
482 }
483
484 #[test]
485 fn attachment_encoding_stops_at_the_upload_budget() {
486 let id = ExGuid {
487 guid: [1; 16],
488 n: 1,
489 };
490 let bytes: Arc<[u8]> = (0..4096)
491 .map(|at| (at % 251) as u8)
492 .collect::<Vec<_>>()
493 .into();
494 let edits = Edits {
495 edits: vec![(
496 "Ada".into(),
497 Edit {
498 at: 0,
499 ops: vec![Op::Page {
500 space: id,
501 op: PageOp::Add {
502 object: PageObject::Attachment(Attachment {
503 id,
504 filename: "payload.bin".into(),
505 source_path: None,
506 size: None,
507 layout: Default::default(),
508 bytes: Some(Arc::clone(&bytes)),
509 preview: None,
510 recording: None,
511 tags: Vec::new(),
512 }),
513 before: None,
514 },
515 }],
516 },
517 )],
518 revisions: BTreeMap::new(),
519 };
520 let encoded = serde_json::to_vec(&edits).unwrap();
521 assert_eq!(encode(&edits, encoded.len()).unwrap(), encoded);
522 let decoded: Edits = serde_json::from_slice(&encoded).unwrap();
523 assert_eq!(decoded.edits, edits.edits);
524 assert_eq!(
525 encode(&edits, encoded.len() - 1).unwrap_err().kind(),
526 io::ErrorKind::FileTooLarge
527 );
528 assert_eq!(
529 encode(&edits, 128).unwrap_err().kind(),
530 io::ErrorKind::FileTooLarge
531 );
532 }
533}