authoravatarclover caruso<git@paperclover.net>2026-10-20 14:10:35-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-21 02:52:29-07:00
logc498049f587516c013f249c7da4903c20cc40045
treed8a1e6e725ab8ad01027b0842b980e4e25efa154
parenta59184893f02f890bfcc387205db09907351ee35
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Preserve dashboard WebSockets across route reloads

Keep upgraded dashboard streams alive for their existing eight-hour session limit when Caddy reloads routes. Remove the unrelated heartbeat and exercise both default and retained streams against a real isolated Caddy process. Assisted-by: gpt-6-astra

3 files changed, 104 insertions(+), 14 deletions(-)

dashboard/src/vms.rs+6-12
...@@ -115,19 +115,13 @@ async fn bridge(socket: WebSocket, stream: tokio::net::UnixStream) {...@@ -115,19 +115,13 @@ async fn bridge(socket: WebSocket, stream: tokio::net::UnixStream) {
115 };115 };
116 let downstream = async {116 let downstream = async {
117 let mut buffer = vec![0; 65536];117 let mut buffer = vec![0; 65536];
118 let mut heartbeat = tokio::time::interval(Duration::from_secs(15));
119 heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
120 loop {118 loop {
121 let message = tokio::select! {119 let count = read.read(&mut buffer).await?;
122 result = read.read(&mut buffer) => {120 if count == 0 {
123 let count = result?;121 break;
124 if count == 0 { break; }122 }
125 Message::Binary(Bytes::copy_from_slice(&buffer[..count]))123 send.send(Message::Binary(Bytes::copy_from_slice(&buffer[..count])))
126 }124 .await
127 _ = heartbeat.tick() => Message::Ping(Bytes::new()),
128 };
129 tokio::time::timeout(Duration::from_secs(5), send.send(message))
130 .await.map_err(std::io::Error::other)?
131 .map_err(std::io::Error::other)?;125 .map_err(std::io::Error::other)?;
132 }126 }
133 Ok::<_, std::io::Error>(())127 Ok::<_, std::io::Error>(())
tools/router-stream-test.py created+94
...@@ -0,0 +1,94 @@
1#!/usr/bin/env python3
2import argparse
3import base64
4import hashlib
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import os
7from pathlib import Path
8import socket
9import subprocess
10import tempfile
11import threading
12import time
13
14import router
15
16
17class Echo(BaseHTTPRequestHandler):
18 protocol_version = "HTTP/1.1"
19
20 def do_GET(self):
21 self.close_connection = True
22 key = self.headers["Sec-WebSocket-Key"] + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
23 self.send_response(101)
24 self.send_header("Upgrade", "websocket")
25 self.send_header("Connection", "Upgrade")
26 self.send_header("Sec-WebSocket-Accept", base64.b64encode(hashlib.sha1(key.encode()).digest()).decode())
27 self.end_headers()
28 while header := self.rfile.read(2):
29 if len(header) != 2 or header[0] != 0x81:
30 break
31 size = header[1] & 127
32 assert header[1] & 128 and size < 126
33 mask = self.rfile.read(4)
34 data = self.rfile.read(size)
35 self.wfile.write(bytes([0x81, size]) + bytes(value ^ mask[index % 4] for index, value in enumerate(data)))
36
37 def log_message(self, *_):
38 pass
39
40
41def free_port():
42 with socket.socket() as listener:
43 listener.bind(("127.0.0.1", 0))
44 return listener.getsockname()[1]
45
46
47def main():
48 parser = argparse.ArgumentParser()
49 parser.add_argument("--caddy", required=True)
50 args = parser.parse_args()
51 with tempfile.TemporaryDirectory(prefix="snowglobe-router-stream-") as temporary, ThreadingHTTPServer(("127.0.0.1", 0), Echo) as backend:
52 root = Path(temporary)
53 threading.Thread(target=backend.serve_forever, daemon=True).start()
54 for keep in (False, True):
55 port, admin = free_port(), free_port()
56 config = root / "Caddyfile"
57 content = "\n".join(["{", f" admin 127.0.0.1:{admin}", " auto_https off", "}", f"http://127.0.0.1:{port} {{",
58 *router.proxy(f"127.0.0.1:{backend.server_port}", " ", keep_streams=keep), "}"])
59 config.write_text(content)
60 environment = {**os.environ, "XDG_DATA_HOME": str(root / "data"), "XDG_CONFIG_HOME": str(root / "config")}
61 with (root / "caddy.log").open("w") as log:
62 process = subprocess.Popen([args.caddy, "run", "--config", str(config)], stdout=log, stderr=log, env=environment)
63 try:
64 deadline = time.monotonic() + 10
65 while True:
66 try:
67 client = socket.create_connection(("127.0.0.1", port), timeout=2)
68 break
69 except OSError:
70 assert process.poll() is None and time.monotonic() < deadline, (root / "caddy.log").read_text()
71 time.sleep(.05)
72 with client, client.makefile("rb") as incoming:
73 client.sendall(f"GET / HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: c3RyZWFtLWZpeHR1cmUtMQ==\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
74 assert b"101" in incoming.readline()
75 while incoming.readline() != b"\r\n":
76 pass
77 client.sendall(b"\x81\x84\0\0\0\0ping")
78 assert incoming.read(6) == b"\x81\x04ping"
79 config.write_text(content.replace(" reverse_proxy", " header X-Reload verified\n reverse_proxy"))
80 subprocess.run([args.caddy, "reload", "--config", str(config), "--address", f"127.0.0.1:{admin}"], check=True, capture_output=True, env=environment)
81 client.sendall(b"\x81\x84\0\0\0\0pong")
82 header = incoming.read(2)
83 assert header == (b"\x81\x04" if keep else b"\x88\x02"), header
84 if keep:
85 assert incoming.read(4) == b"pong"
86 finally:
87 process.terminate()
88 process.wait(timeout=5)
89 backend.shutdown()
90 print("Caddy reload closes default WebSockets and preserves dashboard streams.")
91
92
93if __name__ == "__main__":
94 main()
tools/router.py+4-2
...@@ -28,13 +28,15 @@ def passive_health(upstreams, indent):...@@ -28,13 +28,15 @@ def passive_health(upstreams, indent):
28 return [f"{indent}fail_duration 30s"] if len(upstreams.split()) > 1 else []28 return [f"{indent}fail_duration 30s"] if len(upstreams.split()) > 1 else []
2929
3030
31def proxy(upstreams, indent, uncompressed=False, host=None):31def proxy(upstreams, indent, uncompressed=False, host=None, keep_streams=False):
32 lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s",32 lines = [f"{indent}reverse_proxy {upstreams} {{", f"{indent} lb_try_duration 5s",
33 *passive_health(upstreams, indent + " ")]33 *passive_health(upstreams, indent + " ")]
34 if uncompressed:34 if uncompressed:
35 lines.append(f"{indent} header_up Accept-Encoding identity")35 lines.append(f"{indent} header_up Accept-Encoding identity")
36 if host:36 if host:
37 lines += [f"{indent} header_up Host {host}", f"{indent} header_up X-Forwarded-Host {host}"]37 lines += [f"{indent} header_up Host {host}", f"{indent} header_up X-Forwarded-Host {host}"]
38 if keep_streams:
39 lines.append(f"{indent} stream_close_delay 8h")
38 return [*lines, f"{indent}}}"]40 return [*lines, f"{indent}}}"]
3941
4042
...@@ -465,7 +467,7 @@ def render(token):...@@ -465,7 +467,7 @@ def render(token):
465 lines += [" handle {", " request_header -User-Name", " request_header -User-Groups",467 lines += [" handle {", " request_header -User-Name", " request_header -User-Groups",
466 f" request_header Snowglobe-Proxy-Token {dashboard_proof}",468 f" request_header Snowglobe-Proxy-Token {dashboard_proof}",
467 " request_header X-Snowglobe-Client-IP {remote_host}",469 " request_header X-Snowglobe-Client-IP {remote_host}",
468 *proxy(f"127.0.0.1:{dashboard_port}", " "), " }", "}"]470 *proxy(f"127.0.0.1:{dashboard_port}", " ", keep_streams=True), " }", "}"]
469 internal_port = os.environ.get("SNOWGLOBE_INTERNAL_PORT")471 internal_port = os.environ.get("SNOWGLOBE_INTERNAL_PORT")
470 if internal_port is not None:472 if internal_port is not None:
471 internal_host = "dashboard.internal." + os.environ["SNOWGLOBE_DOMAIN"]473 internal_host = "dashboard.internal." + os.environ["SNOWGLOBE_DOMAIN"]