1//! Meeting peers through a relay (`crates/relay`): one WebSocket to the room, carrying
2//! streams to peers in it, each opened with SPAKE2 and sealed end to end as on a LAN, so the
3//! relay sees only who talks to whom, when, and how much. In a code's room every two peers
4//! have a stream. In a notebook's room only a host and each guest do; everything else goes
5//! to the room's group (`group`), sent once and copied by the relay. A stream or group frame
6//! that breaks makes this end join the room again, which ends every stream it had there;
7//! peers then meet afresh.
8
9use super::{
10 Event, Hello, OPENING, Paced, Peer, Pipe, Presence, Relayed as Answer, Room, Shared, Side,
11 code_parts, group,
12 transport::{self, Address, Failure, parse},
13 wire::kind,
14};
15use ::relay::{BROADCAST, GROUP, Notice, SLOT, Verdict, ws};
16use std::{
17 collections::{HashMap, HashSet, hash_map::Entry},
18 io::{self, BufReader, Read, Write},
19 sync::{Arc, Mutex, atomic::Ordering, mpsc},
20 thread,
21 time::{Duration, Instant},
22};
23
24/// How often a quiet connection pings the relay.
25const KEEPALIVE: Duration = Duration::from_secs(30);
26/// A connection that lasted this long was no failure, so the next waits only a second.
27const STEADY: Duration = Duration::from_secs(60);
28/// The longest wait before joining again, so the room is back soon after a relay restarts.
29const REJOIN: Duration = Duration::from_secs(5);
30/// The most sent in one message, well under any relay's cap.
31const CHUNK: usize = 64 << 10;
32/// The largest message read from the relay.
33const MOST: usize = 1 << 20;
34
35type Reader = ws::Reader<BufReader<Box<dyn Read + Send>>>;
36
37/// Joins `shared`'s room at the relay at `url`, and keeps joining whenever it falls out;
38/// `port` is where it listens, to advertise once the relay numbers its code.
39pub(super) fn join(shared: &Arc<Shared>, url: &str, port: u16) -> io::Result<()> {
40 let address = parse(url)?;
41 let shared = Arc::clone(shared);
42 thread::Builder::new()
43 .name("live relay".into())
44 .spawn(move || keep(&shared, &address, port))?;
45 Ok(())
46}
47
48/// Records how the relay answered, telling `shared`'s events where it changed.
49fn answered(shared: &Arc<Shared>, answer: Answer) {
50 let mut state = shared.state.lock().unwrap();
51 if state.relayed.as_ref() != Some(&answer) {
52 state.relayed = Some(answer);
53 drop(state);
54 (shared.events)(shared, Event::Changed);
55 }
56}
57
58/// Stays in the room at `address`, joining again whenever the connection ends, until
59/// `shared` stops.
60fn keep(shared: &Arc<Shared>, address: &Address, port: u16) {
61 let mut wait = Duration::from_secs(1);
62 let owner = shared.room.owner();
63 // The number of this end's code, asked for again on joining again.
64 let mut nameplate = owner
65 .then(|| code_parts(shared.state.lock().unwrap().code.as_deref()?).0)
66 .flatten();
67 while !shared.stopped.load(Ordering::Acquire) {
68 let began = Instant::now();
69 // The least the relay asked this end to wait.
70 let mut asked = Duration::ZERO;
71 let path = match (owner, shared.tag()) {
72 (false, Some(tag)) => format!("{}/v1/room/{tag}", address.path),
73 (false, None) => return,
74 (true, _) => match nameplate {
75 Some(number) => format!("{}/v1/claim?nameplate={number}", address.path),
76 None => format!("{}/v1/claim", address.path),
77 },
78 };
79 match connect(shared, address, &path, owner) {
80 Ok((socket, reader)) => {
81 shared.state.lock().unwrap().relay = Some(Arc::clone(&socket));
82 answered(shared, Answer::Joined);
83 if !shared.stopped.load(Ordering::Acquire) {
84 session(shared, &socket, reader, &mut nameplate, port);
85 }
86 shared.state.lock().unwrap().relay = None;
87 socket.hang_up();
88 }
89 Err(Failure::Refused(status, retry)) => {
90 eprintln!("Live: the relay refused to let this end in ({status})");
91 answered(shared, Answer::Refused(status, retry));
92 if status == 410 {
93 return;
94 }
95 asked = retry.unwrap_or_default();
96 }
97 Err(Failure::Trouble(trouble, error)) => {
98 eprintln!("Live: no relay at {}: {error}", address.authority);
99 answered(shared, Answer::Unreachable(trouble));
100 }
101 }
102 if began.elapsed() >= STEADY {
103 wait = Duration::from_secs(1);
104 }
105 thread::sleep(pause(wait, asked));
106 wait = (wait * 2).min(REJOIN);
107 }
108}
109
110/// How long to wait before joining again: between half `wait` and all of it, so peers a
111/// restart hung up on come back apart, and never less than the relay `asked`.
112fn pause(wait: Duration, asked: Duration) -> Duration {
113 let mut random = [0; 4];
114 let _ = getrandom::fill(&mut random);
115 let share = f64::from(u32::from_le_bytes(random)) / f64::from(u32::MAX);
116 wait.mul_f64(0.5 + share / 2.0).max(asked)
117}
118
119/// Opens the relay's `path` at `address`: the connection, and what reads its messages.
120fn connect(
121 shared: &Shared,
122 address: &Address,
123 path: &str,
124 owner: bool,
125) -> Result<(Arc<Socket>, Reader), Failure> {
126 let connection = transport::connect(address, path)?;
127 let reader = ws::Reader::new(BufReader::new(connection.reader), MOST, false);
128 let group = matches!(shared.room, Room::Notebook(_)).then(|| Group {
129 keys: group::Keys::new(&shared.secret),
130 members: Mutex::default(),
131 out: Mutex::default(),
132 });
133 let socket = Arc::new(Socket {
134 send: Mutex::new(connection.writer),
135 close: connection.close,
136 owner,
137 links: Mutex::default(),
138 group,
139 });
140 Ok((socket, reader))
141}
142
143/// One connection to a relay's room.
144pub(super) struct Socket {
145 send: Mutex<Box<dyn Write + Send>>,
146 close: Box<dyn Fn() + Send + Sync>,
147 /// Whether this end claimed the room for its code, and so tells the relay who knew it.
148 owner: bool,
149 links: Mutex<Links>,
150 /// A notebook's room's group.
151 pub(super) group: Option<Group>,
152}
153
154/// A notebook's room's group: its key, the peers heard in it, and the way to this end's
155/// thread sending to it.
156pub(super) struct Group {
157 keys: group::Keys,
158 /// The peers heard in the group, by slot.
159 pub(super) members: Mutex<HashMap<u32, group::Member>>,
160 /// To the thread sending this end's group frames, while connected.
161 out: Mutex<Option<mpsc::Sender<Out>>>,
162}
163
164/// What this end sends the group next.
165pub(super) enum Out {
166 /// The newest presence, to everyone.
167 Presence,
168 /// A message kind and its encoded body, to everyone or to the peers in these slots.
169 Frame(u16, Vec<u8>, Option<Vec<u32>>),
170}
171
172impl Group {
173 pub(super) fn send(&self, out: Out) {
174 if let Some(sender) = &*self.out.lock().unwrap() {
175 let _ = sender.send(out);
176 }
177 }
178
179 /// The slot of the member that is peer `id`.
180 pub(super) fn slot(&self, id: &[u8; 16]) -> Option<u32> {
181 let members = self.members.lock().unwrap();
182 members
183 .iter()
184 .find_map(|(slot, member)| (member.hello()?.peer == *id).then_some(*slot))
185 }
186}
187
188/// The streams a socket carries, by the slot of the peer at the other end.
189#[derive(Default)]
190struct Links {
191 /// This end's slot.
192 me: u32,
193 inboxes: HashMap<u32, mpsc::Sender<Vec<u8>>>,
194 /// Slots whose stream ended; a peer that comes back has a new slot.
195 ended: HashSet<u32>,
196}
197
198impl Socket {
199 fn send(&self, opcode: u8, payload: &[u8]) -> io::Result<()> {
200 let mut mask = [0; 4];
201 getrandom::fill(&mut mask).map_err(|_| io::Error::other("System random source failed"))?;
202 let frame = ws::frame(opcode, payload, Some(mask));
203 self.send.lock().unwrap().write_all(&frame)
204 }
205
206 pub(super) fn hang_up(&self) {
207 (self.close)();
208 }
209
210 fn forget(&self, slot: u32) {
211 let mut links = self.links.lock().unwrap();
212 links.inboxes.remove(&slot);
213 links.ended.insert(slot);
214 }
215}
216
217/// Reads notices and peers' bytes from the relay until the connection ends, opening a stream
218/// to each peer in the room.
219fn session(
220 shared: &Arc<Shared>,
221 socket: &Arc<Socket>,
222 mut reader: Reader,
223 nameplate: &mut Option<u32>,
224 port: u16,
225) {
226 let mut tag = shared.tag();
227 // Pings while the session lasts: dropping `_beat` at its end stops them.
228 let (_beat, beats) = mpsc::channel::<()>();
229 let pinging = Arc::clone(socket);
230 thread::spawn(move || {
231 while let Err(mpsc::RecvTimeoutError::Timeout) = beats.recv_timeout(KEEPALIVE) {
232 if pinging.send(ws::PING, &[]).is_err() {
233 return;
234 }
235 }
236 });
237 if let Some(group) = &socket.group {
238 let (out, outgoing) = mpsc::channel();
239 *group.out.lock().unwrap() = Some(out);
240 let (shared, socket) = (Arc::clone(shared), Arc::clone(socket));
241 thread::spawn(move || speak(&shared, &socket, outgoing));
242 }
243 let notebook = socket.group.is_some();
244 while let Ok(message) = reader.read() {
245 match message {
246 ws::Message::Text(text) => match text.parse() {
247 Ok(Notice::Nameplate(number)) => {
248 *nameplate = Some(number);
249 tag = Some(format!("code-{number}"));
250 let super::Room::Code { code, .. } = &shared.room else {
251 continue;
252 };
253 let Some(code) = super::code::format(number, &code_parts(code).1) else {
254 continue;
255 };
256 let mut state = shared.state.lock().unwrap();
257 if state.code.as_ref() != Some(&code) {
258 state.code = Some(code);
259 drop(state);
260 shared.advertise(port);
261 (shared.events)(shared, Event::Changed);
262 }
263 }
264 Ok(Notice::Welcome { you, members }) => {
265 socket.links.lock().unwrap().me = you;
266 for slot in members.into_iter().filter(|_| !notebook) {
267 meet(shared, socket, tag.as_deref(), slot, None);
268 }
269 }
270 Ok(Notice::Joined(slot)) if !notebook => {
271 meet(shared, socket, tag.as_deref(), slot, None);
272 }
273 Ok(Notice::Joined(_)) => {}
274 Ok(Notice::Left(slot)) => {
275 socket.forget(slot);
276 let left = (socket.group.as_ref())
277 .and_then(|group| group.members.lock().unwrap().remove(&slot));
278 if left.is_some() {
279 (shared.events)(shared, Event::Changed);
280 }
281 }
282 Ok(Notice::Burned) => {
283 // Coming back, ask for another number.
284 *nameplate = None;
285 shared.state.lock().unwrap().burned = true;
286 (shared.events)(shared, Event::Changed);
287 }
288 Err(()) => {}
289 },
290 ws::Message::Binary(data) => match data.split_first_chunk::<SLOT>() {
291 Some((slot, frame)) if u32::from_be_bytes(*slot) & GROUP != 0 => {
292 let slot = u32::from_be_bytes(*slot) & !GROUP;
293 heard(shared, socket, tag.as_deref(), slot, frame);
294 }
295 Some((slot, bytes)) => {
296 let slot = u32::from_be_bytes(*slot);
297 meet(shared, socket, tag.as_deref(), slot, Some(bytes.to_vec()));
298 }
299 None => {}
300 },
301 ws::Message::Ping(payload) => {
302 let _ = socket.send(ws::PONG, &payload);
303 }
304 ws::Message::Pong => {}
305 ws::Message::Close => break,
306 }
307 }
308 socket.links.lock().unwrap().inboxes.clear();
309 if let Some(group) = &socket.group {
310 *group.out.lock().unwrap() = None;
311 group.members.lock().unwrap().clear();
312 (shared.events)(shared, Event::Changed);
313 }
314}
315
316/// Sends this end's group frames: its hello, asking everyone for theirs, then its presence
317/// at most every `PRESENCE_EVERY`, and frames as they come.
318fn speak(shared: &Shared, socket: &Socket, outgoing: mpsc::Receiver<Out>) {
319 let Some(group) = &socket.group else {
320 return;
321 };
322 let Ok(mut sealer) = group::Sealer::new(&group.keys) else {
323 return;
324 };
325 let mut send = |kind: u16, body: &[u8], to: Option<&[u32]>| -> io::Result<()> {
326 let sealed = sealer.seal(kind, body, to.is_none())?;
327 let mut message = match to {
328 None => BROADCAST.to_be_bytes().to_vec(),
329 Some(slots) => {
330 let mut message = (GROUP | slots.len() as u32).to_be_bytes().to_vec();
331 slots
332 .iter()
333 .for_each(|slot| message.extend(slot.to_be_bytes()));
334 message
335 }
336 };
337 message.extend(sealed);
338 socket.send(ws::BINARY, &message)
339 };
340 let hello = minicbor::to_vec(&shared.me).unwrap_or_default();
341 if send(kind::HELLO, &hello, None).is_err() {
342 return;
343 }
344 let mut paced = Paced::default();
345 paced.changed();
346 loop {
347 let next = match paced.due {
348 Some(_) => outgoing.recv_timeout(paced.wait(Duration::ZERO)),
349 None => outgoing
350 .recv()
351 .map_err(|_| mpsc::RecvTimeoutError::Disconnected),
352 };
353 let result = match next {
354 Ok(Out::Presence) => {
355 paced.changed();
356 Ok(())
357 }
358 Ok(Out::Frame(kind, body, to)) => send(kind, &body, to.as_deref()),
359 Err(mpsc::RecvTimeoutError::Timeout) => Ok(()),
360 Err(mpsc::RecvTimeoutError::Disconnected) => return,
361 };
362 let result = result.and_then(|()| match paced.due(&shared.state) {
363 Some(presence) => send(
364 kind::PRESENCE,
365 &minicbor::to_vec(&presence).unwrap_or_default(),
366 None,
367 ),
368 None => Ok(()),
369 });
370 if result.is_err() {
371 return;
372 }
373 }
374}
375
376/// Takes in a group frame from the peer in `slot`: its hello, answered with this end's and
377/// meeting it where it serves; its presence; or what else it says.
378fn heard(shared: &Arc<Shared>, socket: &Arc<Socket>, tag: Option<&str>, slot: u32, frame: &[u8]) {
379 let Some(group) = &socket.group else {
380 return;
381 };
382 let opened = {
383 let mut members = group.members.lock().unwrap();
384 let member = match members.entry(slot) {
385 Entry::Occupied(member) => Ok(member.into_mut()),
386 Entry::Vacant(vacant) => {
387 group::Member::new(&group.keys, frame).map(|m| vacant.insert(m))
388 }
389 };
390 member.and_then(|member| {
391 let (kind, body) = member.open(frame)?;
392 Ok((kind, body, member.hello().cloned()))
393 })
394 };
395 let (kind, body, hello) = match opened {
396 Ok(opened) => opened,
397 Err(error) => {
398 eprintln!("Live: the relay broke the room's frames ({error}); meeting again");
399 socket.hang_up();
400 return;
401 }
402 };
403 match kind {
404 kind::HELLO | kind::HELLO_BACK => {
405 let Ok(hello) = minicbor::decode::<Hello>(&body) else {
406 return;
407 };
408 if hello.peer == shared.me.peer {
409 return;
410 }
411 let serves = hello.serves.is_some() && shared.me.serves.is_none();
412 if let Some(member) = group.members.lock().unwrap().get_mut(&slot) {
413 member.peer = Some(Peer {
414 hello: Arc::new(hello),
415 presence: None,
416 });
417 }
418 if kind == kind::HELLO {
419 let presence = minicbor::to_vec(&shared.state.lock().unwrap().presence);
420 let presence = presence.unwrap_or_default();
421 let me = minicbor::to_vec(&shared.me).unwrap_or_default();
422 group.send(Out::Frame(kind::HELLO_BACK, me, Some(vec![slot])));
423 group.send(Out::Frame(kind::PRESENCE, presence, Some(vec![slot])));
424 }
425 if serves {
426 meet(shared, socket, tag, slot, None);
427 }
428 (shared.events)(shared, Event::Changed);
429 }
430 kind::PRESENCE => {
431 let Ok(presence) = minicbor::decode::<Presence>(&body) else {
432 return;
433 };
434 let mut members = group.members.lock().unwrap();
435 if let Some(peer) = members.get_mut(&slot).and_then(|m| m.peer.as_mut()) {
436 peer.presence = Some(presence);
437 drop(members);
438 (shared.events)(shared, Event::Changed);
439 }
440 }
441 // A peer met directly says the rest there.
442 kind if kind < 256 => {
443 if let Some(hello) = hello.filter(|hello| !shared.direct(&hello.peer)) {
444 (shared.events)(
445 shared,
446 Event::Frame {
447 from: &hello,
448 kind,
449 body: &body,
450 },
451 );
452 }
453 }
454 _ => {}
455 }
456}
457
458/// Hands `bytes` from the peer in `slot` to its stream, or opens one. In a code's room this
459/// end opens a stream to a peer with a lower slot when told of it, and answers one with a
460/// higher slot when its first bytes come; in a notebook's room a guest opens one to its host.
461fn meet(
462 shared: &Arc<Shared>,
463 socket: &Arc<Socket>,
464 tag: Option<&str>,
465 slot: u32,
466 bytes: Option<Vec<u8>>,
467) {
468 let mut links = socket.links.lock().unwrap();
469 if let Some(inbox) = links.inboxes.get(&slot) {
470 if let Some(bytes) = bytes {
471 let _ = inbox.send(bytes);
472 }
473 return;
474 }
475 let notebook = socket.group.is_some();
476 let side = match bytes {
477 None if notebook || slot < links.me => Side::Initiator,
478 Some(_) if (notebook && shared.me.serves.is_some()) || (!notebook && slot > links.me) => {
479 Side::Responder
480 }
481 _ => return,
482 };
483 let Some(tag) = tag.filter(|_| !links.ended.contains(&slot)) else {
484 return;
485 };
486 let (inbox, arriving) = mpsc::channel();
487 if let Some(bytes) = bytes {
488 let _ = inbox.send(bytes);
489 }
490 links.inboxes.insert(slot, inbox);
491 let pipe = Arc::new(Relayed {
492 socket: Arc::clone(socket),
493 slot,
494 inbox: Mutex::new(Inbox {
495 arriving,
496 chunk: Vec::new(),
497 at: 0,
498 }),
499 timeout: Mutex::new(OPENING),
500 });
501 let shared = Arc::clone(shared);
502 let tag = tag.to_owned();
503 thread::spawn(move || shared.run(pipe, side, &tag));
504}
505
506/// The stream to one peer through the relay.
507struct Relayed {
508 socket: Arc<Socket>,
509 slot: u32,
510 inbox: Mutex<Inbox>,
511 timeout: Mutex<Duration>,
512}
513
514struct Inbox {
515 arriving: mpsc::Receiver<Vec<u8>>,
516 chunk: Vec<u8>,
517 at: usize,
518}
519
520impl Pipe for Relayed {
521 fn read(&self, buffer: &mut [u8]) -> io::Result<usize> {
522 let mut inbox = self.inbox.lock().unwrap();
523 while inbox.at == inbox.chunk.len() {
524 let timeout = *self.timeout.lock().unwrap();
525 inbox.chunk = match inbox.arriving.recv_timeout(timeout) {
526 Ok(chunk) => chunk,
527 Err(mpsc::RecvTimeoutError::Timeout) => return Err(io::ErrorKind::TimedOut.into()),
528 Err(mpsc::RecvTimeoutError::Disconnected) => return Ok(0),
529 };
530 inbox.at = 0;
531 }
532 let at = inbox.at;
533 let length = buffer.len().min(inbox.chunk.len() - at);
534 buffer[..length].copy_from_slice(&inbox.chunk[at..at + length]);
535 inbox.at += length;
536 Ok(length)
537 }
538
539 fn write(&self, bytes: &[u8]) -> io::Result<usize> {
540 if self.socket.links.lock().unwrap().ended.contains(&self.slot) {
541 return Err(io::ErrorKind::BrokenPipe.into());
542 }
543 let length = bytes.len().min(CHUNK);
544 let message = [&self.slot.to_be_bytes()[..], &bytes[..length]].concat();
545 self.socket.send(ws::BINARY, &message)?;
546 Ok(length)
547 }
548
549 fn set_read_timeout(&self, timeout: Duration) -> io::Result<()> {
550 *self.timeout.lock().unwrap() = timeout;
551 Ok(())
552 }
553
554 fn shutdown(&self) {
555 self.socket.forget(self.slot);
556 }
557
558 fn direct(&self) -> bool {
559 false
560 }
561
562 fn met(&self, met: bool) {
563 if self.socket.owner {
564 let verdict = if met { Verdict::Met } else { Verdict::Failed }(self.slot);
565 let _ = self.socket.send(ws::TEXT, verdict.to_string().as_bytes());
566 }
567 }
568
569 fn broken(&self) {
570 // A deliberate shutdown can wake the reader with EOF.
571 if !self.socket.links.lock().unwrap().ended.contains(&self.slot) {
572 self.socket.hang_up();
573 }
574 }
575}
576
577#[cfg(test)]
578mod tests {
579 use super::*;
580
581 /// However long a relay stays away, the next try comes within seconds of its return,
582 /// tries spread apart, and none comes before the relay's Retry-After.
583 #[test]
584 fn rejoining_waits_a_little_spread_and_as_asked() {
585 let mut wait = Duration::from_secs(1);
586 let mut pauses = Vec::new();
587 for _ in 0..12 {
588 pauses.push(pause(wait, Duration::ZERO));
589 wait = (wait * 2).min(REJOIN);
590 }
591 assert!(
592 pauses.iter().all(|pause| *pause <= Duration::from_secs(5)),
593 "{pauses:?}"
594 );
595 assert!(pauses.windows(2).any(|two| two[0] != two[1]));
596 assert!(pause(wait, Duration::from_secs(30)) >= Duration::from_secs(30));
597 }
598
599 #[test]
600 fn relay_addresses_parse() {
601 let address = parse("wss://live.example.net").unwrap();
602 assert_eq!(
603 address,
604 Address {
605 tls: true,
606 authority: "live.example.net".into(),
607 host: "live.example.net".into(),
608 port: 443,
609 path: String::new(),
610 }
611 );
612 let address = parse("ws://[::1]:7650/snowbound/").unwrap();
613 assert_eq!((address.host.as_str(), address.port), ("::1", 7650));
614 assert_eq!(address.path, "/snowbound");
615 assert!(parse("https://live.example.net").is_err());
616 assert!(parse("ws://:80").is_err());
617 }
618}