1/**
2 * build script for paperclover.net. the root entry point for the site.
3 * @module
4 */
5const root = process.cwd();
6const out = ".clover/o";
7const globalCss = "src/global.css";
8const toRel = (str: string) => path.relative(root, str).replaceAll("\\", "/");
9
10export async function main(graph: incremental.Graph) {
11 if (process.argv.includes("--build")) {
12 await graph.incrementalBuild({ buildAndWriteOutput });
13 return;
14 }
15
16 // in production nothing but a deploy changes the files, so they are read once and not watched
17 const server = await incremental.serve({
18 graph,
19 buildAndWriteOutput,
20 watchFilesystem: !process.argv.includes("--prod"),
21 });
22
23 if (!process.argv.includes("--no-open") && !process.argv.includes("--prod")) {
24 // subprocess.launchBrowser(server.url);
25 }
26
27 // requests in flight finish before the process leaves
28 const stop = () => void server.close().then(() => process.exit(0));
29 process.on("SIGINT", stop);
30 process.on("SIGTERM", stop);
31 for await (const result of server.watch()) {
32 if (result.value) server.setHandler(loadBackend(server, result.value));
33 }
34}
35
36type Respond = (request: Request) => Response | Promise<Response>;
37
38/** hand a build to the code that serves it, loaded by the graph so that it reloads with its code */
39async function loadBackend(
40 server: incremental.Server<Output>,
41 output: Output,
42 progressOverride?: progress.Ref,
43): Promise<Respond> {
44 const { graph } = server;
45 const graphNode = progressOverride ?? server.progress;
46 using _ = graphNode.start("load backend");
47 const sitegen = await graph.import<typeof import("../framework/lib/sitegen.ts")>("framework/lib/sitegen.ts");
48 sitegen.setGraph(graph);
49 const assets = await graph.import<typeof import("../framework/lib/assets.ts")>("framework/lib/assets.ts");
50 await assets.load(output.assets, `${out}/asset`);
51
52 // views are loaded before any request needs one, so serving never waits on the graph
53 const modules: Parameters<typeof view.load>[1] = {};
54 for (const { id, file } of Object.values(output.views)) modules[id] = await graph.import(file);
55 const view = await graph.import<typeof import("../framework/lib/view.ts")>("framework/lib/view.ts");
56 view.load(output, modules);
57
58 const backend = await graph.import<{ default: { fetch: Respond } }>("src/backend.ts");
59 return (req) => {
60 // deployments
61 if (
62 new URL(req.url).pathname === "/.sitegen/rebuild"
63 && !req.headers.has("x-forwarded-for")
64 ) {
65 const root = new progress.Root();
66 graph.incrementalBuild({ buildAndWriteOutput, progress: root })
67 .then((output) => loadBackend(server, output, root))
68 .then((backend) => {
69 server.setHandler(backend);
70 root.end();
71 })
72 .catch((e) => root.error(e));
73 return new Response(progress.encodeByteStream(root), {
74 headers: { "Content-Type": progress.contentType },
75 });
76 }
77
78 return backend.default.fetch(req);
79 };
80}
81
82export interface Output {
83 /** by pathname. the bytes are in `.clover/o/asset`, by hash, also as `.gz` and `.zst` */
84 assets: Record<string, Asset>;
85 /** pages rendered per request, by id */
86 views: Record<string, ts.DeepReadonly<PreparedView>>;
87 /** the step that bundled the client scripts a page can include, by script id */
88 scripts: incremental.Step<{ scripts: Record<string, string> }>;
89 /** the steps that bundled stylesheets, by the style key of a page or view */
90 styles: Record<string, incremental.Step<string>>;
91 /** what every page's metadata starts from */
92 meta: SiteMeta;
93}
94
95export interface Asset {
96 hash: string;
97 headers: Record<string, string>;
98}
99
100export async function buildAndWriteOutput(ref: incremental.Ref, title = "sitegen!"): Promise<Output> {
101 using incr = ref.start(title);
102 await checkSpelling(incr);
103
104 const site = await incr.step(async (b) => {
105 const { siteSections, fonts, meta } = await b.import<typeof import("./site.ts")>("./src/site.ts");
106 // relative paths, so what a build caches still fits once the project has moved
107 const sections = siteSections.map(({ root, base = "/" }) => ({ root: toRel(root), base }));
108 return { sections, fonts, meta };
109 }).read();
110 const files = await discoverAllFiles(incr, site.sections);
111
112 const aliasing = Array.from(Map.groupBy([...files.staticFiles, ...files.pages], (item) => item.id))
113 .filter(([, items]) => items.length > 1);
114 if (aliasing.length > 0) {
115 throw new Error(
116 `source files alias:\n`
117 + aliasing.map(([id, items]) => ` - ${id}\n` + items.map((item) => " - " + item.file).join("\n")).join("\n"),
118 );
119 }
120
121 // -- perform build-time rendering --
122 const fontFiles = site.fonts.flatMap((font) => {
123 const source = incr.step(fonts.fetchFont, font.name, font.sources);
124 return font.subsets.map((subset) => incr.step(fonts.subsetFont, source, subset, site.meta.base));
125 });
126 const staticFiles = files.staticFiles.map((item) => incr.step((b, file) => b.readFile(file).bytes(), item.file));
127 const pages = files.pages.map((item) => incr.step(preparePage, item, site.meta));
128 const views = await Promise.all(files.views.map((item) => incr.step(prepareView, item).read()));
129 const prepared = await Promise.all(pages.map((page) => page.read()));
130 const routes = [...views, ...prepared];
131
132 // -- page resources --
133 const bundleScripts = incr.step(bundle.bundleClientJavaScript, {
134 clientRefs: Array.from(new Set(routes.flatMap((route) => route.clientRefs))).sort(),
135 publicScripts: files.scripts.map((item) => getScriptId(path.resolve(root, item.file))).sort(),
136 });
137 const styles = new Map<string, incremental.Step<string>>();
138 for (const { styleKey, cssImports, theme } of routes) {
139 if (styles.has(styleKey)) continue;
140 styles.set(styleKey, incr.step(css.bundleCssFiles, { cssImports: [...cssImports], theme }));
141 }
142
143 // -- assemble page assets --
144 const documents = pages.map((page, i) =>
145 incr.step(assemblePage, page, UNWRAP(styles.get(prepared[i]!.styleKey)), bundleScripts)
146 );
147
148 // -- write assets, once for all the pathnames with the same bytes --
149 // each asset's bytes are a step of their own, so an asset is hashed and
150 // written only when its bytes change. assets are listed in the order the
151 // files were found, so the same site lists them the same way.
152 const { assets: scriptAssets } = await bundleScripts.read();
153 const fontAssets = await Promise.all(fontFiles.map((font) => font.read()));
154 const html = { "content-type": "text/html;charset=utf8" };
155 const sources: Array<{ pathname: string; bytes: incremental.Step<string | Uint8Array>; headers?: typeof html }> = [
156 ...Object.keys(scriptAssets).map((pathname) => ({
157 pathname,
158 bytes: incr.step(scriptAsset, bundleScripts, pathname),
159 })),
160 ...fontFiles.flatMap((font, i) =>
161 Object.keys(fontAssets[i]!).map((pathname) => ({ pathname, bytes: incr.step(fontAsset, font, pathname) }))
162 ),
163 ...files.staticFiles.map((item, i) => ({ pathname: item.id, bytes: staticFiles[i]! })),
164 ...files.pages.map((item, i) => ({ pathname: item.id, bytes: documents[i]!, headers: html })),
165 ];
166 const hashes = await Promise.all(sources.map(({ bytes }) => incr.step(hashAsset, bytes).read()));
167 const assets: Output["assets"] = {};
168 const writes = new Map<string, Promise<unknown>>();
169 sources.forEach(({ pathname, bytes, headers }, i) => {
170 ASSERT(pathname.startsWith("/") && !(pathname in assets), () => `asset ${pathname} is written twice`);
171 const hash = hashes[i]!;
172 assets[pathname] = {
173 hash,
174 headers: { "content-type": mime.for(pathname), ...headers, etag: JSON.stringify(hash) },
175 };
176 if (!writes.has(hash)) writes.set(hash, incr.step(writeAsset, hash, bytes).read());
177 });
178 await Promise.all(writes.values());
179 await incr.writeFile(`${out}/asset.json`, JSON.stringify(assets));
180
181 // -- types of what a backend can ask for by name --
182 const union = (keys: string[]) => keys.map((key) => JSON.stringify(key)).join(" | ") || "never";
183 const viewTypes = views.map((view) =>
184 ` ${JSON.stringify(view.id)}: typeof import(${JSON.stringify("../../" + view.file)});`
185 );
186 await Promise.all([
187 incr.writeFile(".clover/ts/asset.d.ts", `export type AssetKey = ${union(Object.keys(assets))};\n`),
188 incr.writeFile(".clover/ts/view.d.ts", `export interface RegisteredViews {\n${viewTypes.join("\n")}\n}\n`),
189 ]);
190
191 return {
192 assets,
193 views: Object.fromEntries(views.map((view) => [view.id, view])),
194 scripts: bundleScripts,
195 styles: Object.fromEntries(styles),
196 meta: site.meta,
197 };
198}
199
200async function discoverAllFiles(
201 ref: incremental.Ref,
202 sections: readonly Readonly<sg.Section>[],
203) {
204 using node = ref.start("discover files");
205 const found = await Promise.all(
206 sections.map(({ root, base = "/" }) =>
207 node.step(scanSiteSection, {
208 root,
209 base: base.endsWith("/") ? base : base + "/",
210 }).read()
211 ),
212 );
213 return {
214 staticFiles: found.flatMap((section) => section.staticFiles),
215 pages: found.flatMap((section) => section.pages),
216 views: found.flatMap((section) => section.views),
217 scripts: found.flatMap((section) => section.scripts),
218 };
219}
220
221export async function scanSiteSection(b: incremental.Build, section: Readonly<sg.Section>) {
222 using _ = b.start(section.root);
223 // Static files are compressed and served as-is.
224 // - "{section}/static/*.png"
225 let staticFiles: sg.FileItem[] = [];
226 // Pages are rendered then served as static files.
227 // - "{section}/pages/*.
228 let pages: sg.FileItem[] = [];
229 // Views are dynamically rendered pages called via backend code.
230 // - "{section}/views/*.tsx"
231 let views: sg.FileItem[] = [];
232 // Public scripts are bundled for the client as static assets under "/js/[...]"
233 // This is used for the file viewer's canvases.
234 // Note that '.client.ts' can be placed anywhere in the file structure.
235 // - "{section}/scripts/*.client.ts"
236 let scripts: sg.FileItem[] = [];
237
238 const sectionPath = (...sub: string[]) => path.join(section.root, ...sub);
239 const rootPrefix = section.root === "src"
240 ? ""
241 : path.relative("src", section.root) + "/";
242 const kinds = [
243 {
244 dir: sectionPath("pages"),
245 list: pages,
246 prefix: section.base,
247 include: [".marko", ".mdo"],
248 exclude: [".client.ts"],
249 page: true,
250 },
251 {
252 dir: sectionPath("static"),
253 list: staticFiles,
254 prefix: section.base,
255 ext: true,
256 page: true,
257 },
258 {
259 dir: sectionPath("scripts"),
260 list: scripts,
261 prefix: rootPrefix,
262 include: [".client.ts"],
263 page: false,
264 },
265 {
266 dir: sectionPath("views"),
267 list: views,
268 prefix: rootPrefix,
269 include: [".marko", ".mdo"],
270 exclude: [".client.ts"],
271 page: false,
272 },
273 ];
274 for (const kind of kinds) {
275 const {
276 dir,
277 list,
278 prefix,
279 include = [""],
280 exclude = [],
281 ext = false,
282 } = kind;
283
284 let items;
285 try {
286 items = await b.readDirRecursive(dir);
287 } catch (err) {
288 if (exception.code(err) === "ENOENT") continue;
289 throw err;
290 }
291 for (const subPath of items) {
292 const file = path.join(dir, subPath);
293 if (file.endsWith("/")) continue;
294 if (!include.some((e) => subPath.endsWith(e))) continue;
295 if (exclude.some((e) => subPath.endsWith(e))) continue;
296 const trim = ext
297 ? subPath
298 : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
299 let id = prefix + trim.replaceAll("\\", "/");
300 if (kind.page && id.endsWith("/index")) {
301 id = id.slice(0, -"/index".length) || "/";
302 }
303 list.push({ id, file: toRel(file) });
304 }
305 }
306
307 return { staticFiles, pages, views, scripts };
308}
309
310type Route = sg.ViewExports & { layout?: { default?: unknown; theme?: css.Theme } };
311// what a step can hold of a template: no url, and no function
312type SiteMeta = Omit<meta.Template, "base" | "titleTemplate"> & { base: string };
313
314/** render a page that is the same for every request */
315export async function preparePage(b: incremental.Build, item: Readonly<sg.FileItem>, template: SiteMeta) {
316 using _ = b.start(item.file);
317 try {
318 const { default: Page, meta: metadata, theme: pageTheme, layout } = await b.import<Route>(item.file);
319 if (!Page) throw new Error("Page is missing a 'default' export.");
320 if (!metadata) throw new Error("Page is missing 'meta' export with a title.");
321
322 const theme: css.Theme = { ...css.defaultTheme, ...layout?.theme ?? pageTheme };
323 const cssImports = await findCssImports(b, item.file);
324
325 const renderedMeta = meta.toHtml({ ...template, base: new URL(template.base) }, {
326 canonical: item.id,
327 ...typeof metadata === "function" ? await metadata({ ssr: true }) : metadata,
328 });
329 if (!renderedMeta.includes("<title>")) {
330 throw new Error("Page is missing 'meta.title'. " + "All pages need a title tag.");
331 }
332
333 // components add the scripts they need while rendering. marko would pick a
334 // random id for the render, and a step has to return the same html every time
335 const cloverScripts = new Set<string>();
336 const $global = { renderId: "_" };
337 const html = String(await SitegenWrap.render({ $global, cloverScripts, page: Page, attrs: {} }));
338
339 return {
340 id: item.id,
341 file: item.file,
342 html,
343 meta: renderedMeta,
344 cssImports,
345 theme,
346 styleKey: css.styleKey(cssImports, theme),
347 clientRefs: Array.from(cloverScripts),
348 };
349 } catch (e) {
350 exception.rethrowWithMetadata(e, { page: item.id });
351 }
352}
353
354/** what a server needs beside a view's module to render it per request */
355export async function prepareView(b: incremental.Build, item: Readonly<sg.FileItem>) {
356 using _ = b.start(item.file);
357 const module = await b.import<Partial<Route>>(item.file);
358 if (!module.meta) throw new Error(`View is missing 'export const meta'`);
359 if (!module.default) throw new Error(`View is missing a default export.`);
360 const theme: css.Theme = { ...css.defaultTheme, ...module.layout?.theme ?? module.theme };
361 const cssImports = await findCssImports(b, item.file);
362 // a view renders with props no build knows, so it gets every script it could include
363 const clientRefs = (await b.trackModule(item.file)).flatMap((module) =>
364 (module.data as { scannedClientRefs?: string[] } | null)?.scannedClientRefs ?? []
365 );
366 return {
367 id: item.id,
368 file: item.file,
369 cssImports,
370 theme,
371 clientRefs: Array.from(new Set(clientRefs)),
372 hasLayout: !!module.layout?.default,
373 styleKey: css.styleKey(cssImports, theme),
374 };
375}
376export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
377
378/** the stylesheets a module imports, in import order, after the one every page has */
379async function findCssImports(b: incremental.Build, file: string): Promise<string[]> {
380 const modules = await b.trackModule(file);
381 const imports = new Map(modules.map((module) => [module.url, module.imports]));
382 const found = new Set([globalCss]);
383 const seen = new Set<string>();
384 const visit = (href: string) => {
385 if (seen.has(href)) return;
386 seen.add(href);
387 if (href.endsWith(".css")) return void found.add(toRel(url.fileURLToPath(href)));
388 imports.get(href)?.forEach(visit);
389 };
390 visit(UNWRAP(modules[0]).url);
391 return Array.from(found);
392}
393
394/** the whole html document of a page, with its styles and scripts inline */
395export async function assemblePage(
396 b: incremental.Build,
397 page: incremental.Step<Awaited<ReturnType<typeof preparePage>>>,
398 style: incremental.Step<string>,
399 scripts: incremental.Step<{ scripts: Record<string, string> }>,
400) {
401 const { html, meta, clientRefs } = await b.readStep(page);
402 const scriptMap = (await b.readStep(scripts)).scripts;
403 return sg.wrapDocument({
404 body: html,
405 head: meta,
406 inlineCss: await b.readStep(style),
407 scripts: sg.joinScripts(clientRefs.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))),
408 });
409}
410
411/** one of the scripts the bundle made, so writing it follows that script alone */
412async function scriptAsset(
413 b: incremental.Build,
414 bundle: incremental.Step<{ assets: Record<string, string> }>,
415 pathname: string,
416) {
417 return UNWRAP((await b.readStep(bundle)).assets[pathname]);
418}
419
420/** one of the files of a font subset */
421async function fontAsset(b: incremental.Build, font: incremental.Step<Record<string, Uint8Array>>, pathname: string) {
422 return UNWRAP((await b.readStep(font))[pathname]);
423}
424
425/** the hash that an asset's file is named by */
426async function hashAsset(b: incremental.Build, bytes: incremental.Step<string | Uint8Array>) {
427 return crypto.createHash("sha1").update(await b.readStep(bytes)).digest("hex");
428}
429
430/** an asset's file and its compressed copies, named by its hash */
431async function writeAsset(b: incremental.Build, hash: string, bytes: incremental.Step<string | Uint8Array>) {
432 const content = await b.readStep(bytes);
433 const [gzipped, zstded] = await Promise.all([gzip(content), zstd(content)]);
434 await Promise.all([
435 b.writeFile(`${out}/asset/${hash}`, content),
436 b.writeFile(`${out}/asset/${hash}.gz`, gzipped),
437 b.writeFile(`${out}/asset/${hash}.zst`, zstded),
438 ]);
439}
440
441const gzip = util.promisify(zlib.gzip);
442const zstd = util.promisify(zlib.zstdCompress);
443
444import * as sg from "#sitegen";
445import { ASSERT, UNWRAP } from "@clo/lib/assert";
446import * as exception from "@clo/lib/exception";
447import * as incremental from "@clo/lib/incremental";
448import * as meta from "@clo/lib/meta";
449import * as mime from "@clo/lib/mime";
450import * as progress from "@clo/lib/progress";
451import * as subprocess from "@clo/lib/subprocess";
452import type * as ts from "@clo/lib/ts";
453import * as crypto from "node:crypto";
454import path from "node:path";
455import * as url from "node:url";
456import * as util from "node:util";
457import * as zlib from "node:zlib";
458import * as bundle from "../framework/bundle.ts";
459import * as css from "../framework/css.ts";
460import * as fonts from "../framework/font.ts";
461import { getScriptId } from "../framework/marko.ts";
462import SitegenWrap from "../framework/sitegen-wrap.marko";
463import { checkSpelling } from "./spelling.ts";