authoravatarclover caruso<git@paperclover.net>2026-10-20 14:10:35-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logc498049f587516c013f249c7da4903c20cc40045
treed8a1e6e725ab8ad01027b0842b980e4e25efa154
parenta59184893f02f890bfcc387205db09907351ee35
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Preserve dashboard WebSockets across route reloads

Keep upgraded dashboard streams alive for their existing eight-hour session limit when Caddy reloads routes. Remove the unrelated heartbeat and exercise both default and retained streams against a real isolated Caddy process. Assisted-by: gpt-6-astra

3 files changed, 104 insertions(+), 14 deletions(-)

dashboard/src/vms.rs+6-12
......@@ -115,19 +115,13 @@ async fn bridge(socket: WebSocket, stream: tokio::net::UnixStream) {
115115 };
116116 let downstream = async {
117117 let mut buffer = vec![0; 65536];
118 let mut heartbeat = tokio::time::interval(Duration::from_secs(15));
119 heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
120118 loop {
121 let message = tokio::select! {
122 result = read.read(&mut buffer) => {
123 let count = result?;
124 if count == 0 { break; }
125 Message::Binary(Bytes::copy_from_slice(&buffer[..count]))
126 }
127 _ = heartbeat.tick() => Message::Ping(Bytes::new()),
128 };
129 tokio::time::timeout(Duration::from_secs(5), send.send(message))
130 .await.map_err(std::io::Error::other)?
119 let count = read.read(&mut buffer).await?;
120 if count == 0 {
121 break;
122 }
123 send.send(Message::Binary(Bytes::copy_from_slice(&buffer[..count])))
124 .await
131125 .map_err(std::io::Error::other)?;
132126 }
133127 Ok::<_, std::io::Error>(())
tools/router-stream-test.py created+94
......@@ -0,0 +1,94 @@
1#!/usr/bin/env python3
2import argparse
3import base64
4import hashlib
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import os
7from pathlib import Path
8import socket
9import subprocess
10import tempfile
11import threading
12import time
13
14import router
15
16
17class Echo(BaseHTTPRequestHandler):
18 protocol_version = "HTTP/1.1"
19
20 def do_GET(self):
21 self.close_connection = True
22 key = self.headers["Sec-WebSocket-Key"] + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
23 self.send_response(101)
24 self.send_header("Upgrade", "websocket")
25 self.send_header("Connection", "Upgrade")
26 self.send_header("Sec-WebSocket-Accept", base64.b64encode(hashlib.sha1(key.encode()).digest()).decode())
27 self.end_headers()
28 while header := self.rfile.read(2):
29 if len(header) != 2 or header[0] != 0x81:
30 break
31 size = header[1] & 127
32 assert header[1] & 128 and size < 126
33 mask = self.rfile.read(4)
34 data = self.rfile.read(size)
35 self.wfile.write(bytes([0x81, size]) + bytes(value ^ mask[index % 4] for index, value in enumerate(data)))
36
37 def log_message(self, *_):
38 pass
39
40
41def free_port():
42 with socket.socket() as listener:
43 listener.bind(("127.0.0.1", 0))
44 return listener.getsockname()[1]
45
46
47def main():
48 parser = argparse.ArgumentParser()
49 parser.add_argument("--caddy", required=True)
50 args = parser.parse_args()
51 with tempfile.TemporaryDirectory(prefix="snowglobe-router-stream-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Echo) as backend:
52 root = Path(temporary)
53 threading.Thread(target=backend.serve_forever, daemon=True).start()
54 for keep in (False, True):
55 port, admin = free_port(), free_port()
56 config = root / "Caddyfile"
57 content = "\n".join(["{", f" admin 127.0.0.1:{admin}", " auto_https off", "}", f"http://127.0.0.1:{port} {{",
58 *router.proxy(f"127.0.0.1:{backend.server_port}", " ", keep_streams=keep), "}"])
59 config.write_text(content)
60 environment = {**os.environ, "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")}
61 with (root / "caddy.log").open("w") as log:
62 process = subprocess.Popen([args.caddy, "run", "--config", str(config)], stdout=log, stderr=log, env=environment)
63 try:
64 deadline = time.monotonic() + 10
65 while True:
66 try:
67 client = socket.create_connection(("127.0.0.1", port), timeout=2)
68 break
69 except OSError:
70 assert process.poll() is None and time.monotonic() < deadline, (root / "caddy.log").read_text()
71 time.sleep(.05)
72 with client, client.makefile("rb") as incoming:
73 client.sendall(f"GET / HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: c3RyZWFtLWZpeHR1cmUtMQ==\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
74 assert b"101" in incoming.readline()
75 while incoming.readline() != b"\r\n":
76 pass
77 client.sendall(b"\x81\x84\0\0\0\0ping")
78 assert incoming.read(6) == b"\x81\x04ping"
79 config.write_text(content.replace(" reverse_proxy", " header X-Reload verified\n reverse_proxy"))
80 subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment)
81 client.sendall(b"\x81\x84\0\0\0\0pong")
82 header = incoming.read(2)
83 assert header == (b"\x81\x04" if keep else b"\x88\x02"), header
84 if keep:
85 assert incoming.read(4) == b"pong"
86 finally:
87 process.terminate()
88 process.wait(timeout=5)
89 backend.shutdown()
90 print("Caddy reload closes default WebSockets and preserves dashboard streams.")
91
92
93if __name__ == "__main__":
94 main()
tools/router.py+4-2
......@@ -28,13 +28,15 @@ def passive_health(upstreams, indent):
2828 return [f"{indent}fail_duration 30s"] if len(upstreams.split()) > 1 else []
2929
3030
31def proxy(upstreams, indent, uncompressed=False, host=None):
31def proxy(upstreams, indent, uncompressed=False, host=None, keep_streams=False):
3232 lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s",
3333 *passive_health(upstreams, indent + " ")]
3434 if uncompressed:
3535 lines.append(f"{indent} header_up Accept-Encoding identity")
3636 if host:
3737 lines += [f"{indent} header_up Host {host}", f"{indent} header_up X-Forwarded-Host {host}"]
38 if keep_streams:
39 lines.append(f"{indent} stream_close_delay 8h")
3840 return [*lines, f"{indent}}}"]
3941
4042
......@@ -465,7 +467,7 @@ def render(token):
465467 lines += [" handle {", " request_header -User-Name", " request_header -User-Groups",
466468 f" request_header Snowglobe-Proxy-Token {dashboard_proof}",
467469 " request_header X-Snowglobe-Client-IP {remote_host}",
468 *proxy(f"127.0.0.1:{dashboard_port}", " "), " }", "}"]
470 *proxy(f"127.0.0.1:{dashboard_port}", " ", keep_streams=True), " }", "}"]
469471 internal_port = os.environ.get("SNOWGLOBE_INTERNAL_PORT")
470472 if internal_port is not None:
471473 internal_host = "dashboard.internal." + os.environ["SNOWGLOBE_DOMAIN"]