1// Keeps the web node's copy of cache.sqlite in sync with the source of
2// truth. The SOT serves a server-sent-events feed at /db/events; this
3// module holds it open and pulls /db on every beacon. The server emits a
4// beacon on connect, so boot, reconnect, and missed-while-disconnected all
5// resolve through the same path — there is no polling and no TTL.
6//
7// Pulls are ETag-guarded (no-change is a tiny 304; the etag is persisted so
8// a reboot does not re-download an unchanged database), and land through
9// `db.restore`, so every open connection reads the new rows.
10const console = log.scoped("sync");
11
12const token = process.env.CLOVER_SOT_KEY ?? null;
13const enabled = process.env.CLOVER_DB_SYNC !== "0" && token != null;
14const sotUrl = new URL(process.env.CLOVER_SOT_URL ?? "https://db.paperclover.net")
15 .toString().replace(/\/+$/g, "");
16if (!enabled) {
17 console.warn(
18 "database sync disabled"
19 + (token ? " (CLOVER_DB_SYNC=0)" : " (no CLOVER_SOT_KEY)"),
20 );
21}
22
23let inFlight: Promise<void> | null = null;
24
25function revalidate(): Promise<void> {
26 return inFlight ??= revalidateInner().finally(() => {
27 inFlight = null;
28 });
29}
30
31async function revalidateInner(): Promise<void> {
32 const res = await fetch(`${sotUrl}/db`, {
33 headers: {
34 Authorization: token!,
35 ...etag() ? { "If-None-Match": etag()! } : null,
36 },
37 signal: AbortSignal.timeout(120_000),
38 });
39 if (res.status === 304) return;
40 if (!res.ok || !res.body) {
41 throw new Error(`source of truth responded ${res.status} ${res.statusText}`);
42 }
43 await mediaFiles.db.restore(res.body);
44 saveEtag(res.headers.get("ETag"));
45 console.info(`database updated (${string.formatByteSize(Number(res.headers.get("Content-Length") ?? "0"))})`);
46}
47
48// -- the persisted etag, stored beside the database --
49let cachedEtag: string | null | undefined;
50function etagPath() {
51 return mediaFiles.db.location + ".etag";
52}
53function etag(): string | null {
54 if (cachedEtag === undefined) {
55 try {
56 cachedEtag = fs.readFileSync(etagPath(), "utf-8").trim() || null;
57 } catch {
58 cachedEtag = null;
59 }
60 }
61 return cachedEtag;
62}
63function saveEtag(value: string | null) {
64 cachedEtag = value;
65 try {
66 fs.writeFileSync(etagPath(), value ?? "");
67 } catch (err) {
68 console.warn("could not persist database etag:", err);
69 }
70}
71
72// -- the subscription --
73// a hand-rolled sse reader over fetch (EventSource cannot send the
74// Authorization header). any event block triggers a revalidate; a failed
75// revalidate aborts the connection so the reconnect path retries it.
76async function subscribeLoop(stop: AbortSignal) {
77 let backoff = 1000;
78 while (!stop.aborted) {
79 const ctrl = new AbortController();
80 let watchdog: NodeJS.Timeout | null = null;
81 try {
82 const res = await fetch(`${sotUrl}/db/events`, {
83 headers: { Authorization: token! },
84 signal: AbortSignal.any([stop, ctrl.signal]),
85 });
86 if (!res.ok || !res.body) {
87 throw new Error(`${res.status} ${res.statusText}`);
88 }
89 console.info(`subscribed to database changes from ${sotUrl}`);
90 backoff = 1000;
91
92 // the server keepalives every 25s; 90s of silence means the
93 // connection died without a FIN
94 const armWatchdog = () => {
95 if (watchdog) clearTimeout(watchdog);
96 watchdog = setTimeout(
97 () => ctrl.abort(new Error("no keepalive")),
98 90_000,
99 );
100 watchdog.unref();
101 };
102 armWatchdog();
103 const reader = res.body.getReader();
104 const decoder = new TextDecoder();
105 let buffer = "";
106 while (true) {
107 const { value, done } = await reader.read();
108 if (done) break;
109 armWatchdog();
110 buffer += decoder.decode(value, { stream: true });
111 const blocks = buffer.split("\n\n");
112 buffer = blocks.pop()!;
113 const changed = blocks.some((block) =>
114 block.split("\n").some((line) => line.startsWith("event:") || line.startsWith("data:"))
115 );
116 if (changed) {
117 void revalidate().catch((err) => {
118 console.error("revalidate failed:", err);
119 ctrl.abort(new Error("revalidate failed"));
120 });
121 }
122 }
123 throw new Error("stream ended");
124 } catch (err: any) {
125 if (stop.aborted) return;
126 console.warn(
127 `database subscription lost (${err?.message ?? err}); `
128 + `retrying in ${Math.round(backoff / 1000)}s`,
129 );
130 } finally {
131 if (watchdog) clearTimeout(watchdog);
132 ctrl.abort();
133 }
134 await new Promise<void>((resolve) => setTimeout(resolve, backoff).unref());
135 backoff = Math.min(backoff * 2, 60_000);
136 }
137}
138
139if (enabled) {
140 // reloading this code subscribes again, so the old subscription ends
141 const stop = new AbortController();
142 incremental.onDispose(import.meta, () => stop.abort());
143 void subscribeLoop(stop.signal);
144}
145
146import * as fs from "node:fs";
147
148import * as mediaFiles from "#src/file-viewer/models/media-files.ts";
149
150import * as incremental from "@clo/lib/incremental";
151import * as log from "@clo/lib/log";
152import * as string from "@clo/lib/string";