build.ts
17.1 kB · typescript · 478 lines
1import { createHash } from "node:crypto";2import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";3import { basename, dirname, extname, join, normalize, relative, resolve } from "node:path";4import { deflateSync } from "node:zlib";5import { collect as gitRoutes, forest, isGit, print as gitPrint, render as gitRender, type Hooks } from "../git/git.ts";6import { index, stamp, type Index } from "./links.ts";7import { grid, logoSvg } from "../ui/logo.js";89/* TYPES */1011export type Bytes = string | Uint8Array;1213export type Output = { path: string; bytes: Bytes; type?: string };1415export type Link = { route: string; name?: string; at?: string; source?: string };1617export type Route = {18 route: string;19 kind?: string;20 name?: string;21 data?: unknown;22 source?: string;23 inputs?: string[];24 urls?: Link[];25 at?: string;26 hidden?: boolean;27 sitemap?: boolean;28};2930export type Node = { name: string; href?: string; nodes?: Node[]; open?: boolean; lazy?: string; icon?: string; figure?: { dark: string; light: string }; text?: string; dates?: string[] };3132export type Input = { name: string; path: string; files: string[]; missing: boolean };3334export type Bundle = { path: string; out: string; hash: boolean; files: string[]; ext?: string };3536export type Site = {37 root: string;38 out: string;39 config: Config;40 inputs: Record<string, Input>;41 kit: Bundle | null;42 routes: Route[];43 nav: Node[];44 stamp: string;45 index: Index | null;46 asset: (name: string) => string;47 input: (name: string) => Input;48 bytes: (file: string) => Uint8Array;49};5051export type Config = {52 title?: string;53 root?: string;54 inputs?: Record<string, { path: string; ext?: string; deep?: boolean }>;55 kit?: { path: string; out?: string; hash?: boolean; files?: string[]; ext?: string };56 assets?: { path: string; out?: string; hash?: boolean; files?: string[]; ext?: string }[];57 manifest?: Record<string, unknown>;58 robots?: { disallow?: string[] };59 llms?: { about?: string; links?: { href: string; name?: string; note?: string }[] };60 [key: string]: unknown;61};6263export type Spec = {64 root: string;65 out: string;66 config?: Config;67 templates?: string[];68 collect: (site: Site) => Promise<{ routes: Route[]; nav?: Node[] }> | { routes: Route[]; nav?: Node[] };69 render: (site: Site, route: Route) => Promise<Output[]> | Output[];70 globals?: (site: Site) => Promise<Output[]> | Output[];71 git?: Hooks;72 asset?: (name: string, body: Uint8Array) => Bytes;73};7475export type Record_ = { hash: string; at: string; outputs: string[]; types?: Record<string, string> };7677export type Manifest = Record<string, Record_>;7879/* FILES */8081const SKIP = new Set(["node_modules", "dist", ".git", ".cache", "target", "data", "pkg"]);8283export function walk(dir: string, deep = true): string[] {84 if (!existsSync(dir)) return [];85 const out: string[] = [];86 for (const item of readdirSync(dir, { withFileTypes: true })) {87 if (item.name.startsWith(".")) continue;88 const path = join(dir, item.name);89 if (item.isDirectory()) {90 if (deep && !SKIP.has(item.name)) out.push(...walk(path, deep));91 } else out.push(path);92 }93 return out.sort();94}9596const cache = new Map<string, Uint8Array>();9798export function forget() {99 cache.clear();100}101102export function bytes(file: string): Uint8Array {103 const hit = cache.get(file);104 if (hit) return hit;105 const data = new Uint8Array(readFileSync(file));106 cache.set(file, data);107 return data;108}109110const digest = (parts: Bytes[]) => {111 const h = createHash("sha256");112 for (const part of parts) h.update(part);113 return h.digest("hex");114};115116const short = (text: string) => text.slice(0, 8);117118const escape = (text: string) =>119 text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);120121/* SCAN */122123function inputs(root: string, config: Config): Record<string, Input> {124 const found: Record<string, Input> = {};125 for (const [name, decl] of Object.entries(config.inputs ?? {})) {126 const path = resolve(root, decl.path);127 const all = existsSync(path) && statSync(path).isFile() ? [path] : walk(path, decl.deep ?? false);128 const files = decl.ext ? all.filter((f) => f.endsWith(decl.ext!)) : all;129 found[name] = { name, path, files, missing: !existsSync(path) };130 }131 return found;132}133134function templates(spec: Spec): string {135 const here = resolve(import.meta.dir, "..");136 const dirs = [here, ...(spec.templates ?? []).map((d) => resolve(spec.root, d))];137 const parts: Bytes[] = [];138 for (const dir of dirs) for (const file of walk(dir)) parts.push(relative(dir, file), bytes(file));139 return digest(parts);140}141142function bundle(root: string, decl: NonNullable<Config["kit"]>): Bundle {143 const path = resolve(root, decl.path);144 const named = decl.files ?? walk(path, false).map((f) => f.slice(path.length + 1));145 const files = decl.ext ? named.filter((f) => f.endsWith(decl.ext!)) : named;146 return { path, out: (decl.out ?? "ui").replace(/^\/|\/$/g, ""), hash: decl.hash ?? false, files };147}148149function bundles(root: string, config: Config): Bundle[] {150 const list: Bundle[] = [];151 if (config.kit) list.push(bundle(root, config.kit));152 for (const decl of config.assets ?? []) list.push(bundle(root, decl));153 return list;154}155156/* ASSETS */157158const IMPORT = /((?:\bfrom|\bimport)\s*\(?\s*)(["'])(\.\.?\/[^"']+)\2/g;159160function place(one: Bundle, spec: Spec, assets: Map<string, string>, copies: Output[]) {161 const known = new Set(one.files);162 const busy = new Set<string>();163 const visit = (name: string): string => {164 const hit = assets.get(name);165 if (hit) return hit;166 if (busy.has(name)) throw new Error(`ssg: ${name} imports itself around a cycle`);167 busy.add(name);168 const file = join(one.path, name);169 if (!existsSync(file)) throw new Error(`ssg: asset missing: ${file}`);170 const raw = bytes(file);171 let body = spec.asset ? spec.asset(name, raw) : raw;172 if (one.hash && /\.m?js$/.test(name)) {173 const text = typeof body === "string" ? body : new TextDecoder().decode(body);174 body = text.replace(IMPORT, (whole, head: string, quote: string, target: string) => {175 const dep = normalize(join(dirname(name), target));176 if (known.has(dep)) return `${head}${quote}${visit(dep)}${quote}`;177 if (existsSync(join(one.path, dep))) throw new Error(`ssg: ${name} imports ${dep}, which the kit's files do not list`);178 return whole;179 });180 }181 const data = typeof body === "string" ? new TextEncoder().encode(body) : body;182 const stem = name.replace(/(\.[^.]+)$/, "");183 const path = one.hash ? `${one.out}/${stem}-${short(digest([data]))}${extname(name)}` : `${one.out}/${name}`;184 assets.set(name, `/${path}`);185 copies.push({ path, bytes: data });186 busy.delete(name);187 return `/${path}`;188 };189 for (const name of one.files) visit(name);190}191192const shows = (nodes: Node[], href: string): boolean =>193 nodes.some((node) => node.href === href || shows(node.nodes ?? [], href));194195export async function scan(spec: Spec): Promise<Site> {196 const root = resolve(spec.root);197 const config = spec.config ?? (JSON.parse(readFileSync(join(root, "site.json"), "utf8")) as Config);198 const found = inputs(root, config);199 const list = bundles(root, config);200 const kit = list[0] ?? null;201 const assets = new Map<string, string>();202 const copies: Output[] = [];203 for (const one of list) place(one, spec, assets, copies);204 const site: Site = {205 root,206 out: resolve(spec.out),207 config,208 inputs: found,209 kit,210 routes: [],211 nav: [],212 stamp: "",213 index: null,214 asset: (name) => {215 const hit = assets.get(name);216 if (!hit) throw new Error(`ssg: no asset named ${name}`);217 return hit;218 },219 input: (name) => {220 const hit = found[name];221 if (!hit) throw new Error(`ssg: ${name} is not declared under inputs in site.json`);222 return hit;223 },224 bytes,225 };226 const picked = await spec.collect(site);227 site.routes = picked.routes;228 site.nav = picked.nav ?? [];229 const repo = gitRoutes(site);230 if (repo.routes.length) {231 site.routes = [...site.routes, ...repo.routes];232 if (repo.node && !shows(site.nav, repo.node.href!)) site.nav = [...site.nav, repo.node];233 }234 site.index = index(site);235 site.stamp = digest([236 templates(spec),237 JSON.stringify(site.nav),238 JSON.stringify(config),239 JSON.stringify([...assets]),240 stamp(site.index),241 ]);242 (site as { copies?: Output[] }).copies = copies;243 return site;244}245246/* FINGERPRINT */247248export function label(site: Site, file: string): string {249 let base = "";250 let name = "";251 for (const one of Object.values(site.inputs)) {252 if (one.path.length <= base.length) continue;253 if (file !== one.path && !file.startsWith(`${one.path}/`)) continue;254 base = one.path;255 name = one.name;256 }257 if (!base) return basename(file);258 return file === base ? name : `${name}/${file.slice(base.length + 1)}`;259}260261export function fingerprint(site: Site, route: Route): string {262 if (isGit(route)) return gitPrint(site, route);263 const files = route.inputs ?? (route.source ? [route.source] : []);264 const named = files.map((file) => [label(site, file), file] as const).sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));265 const parts: Bytes[] = [route.route, route.kind ?? "", JSON.stringify(route.data ?? null), site.stamp];266 for (const [name, file] of named) {267 parts.push(name);268 if (!existsSync(file)) {269 parts.push("gone");270 continue;271 }272 if (statSync(file).isDirectory()) for (const inner of walk(file)) parts.push(relative(file, inner), bytes(inner));273 else parts.push(bytes(file));274 }275 return digest(parts).slice(0, 16);276}277278/* RENDER */279280export async function render(site: Site, route: Route, spec: Spec): Promise<Output[]> {281 if (isGit(route)) return gitRender(site, route, spec);282 return await spec.render(site, route);283}284285/* ICONS */286287const SIGNATURE = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10]);288289const TABLE = new Uint32Array(256).map((_, n) => {290 let c = n;291 for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;292 return c >>> 0;293});294295function crc32(data: Uint8Array): number {296 let c = 0xffffffff;297 for (const b of data) c = TABLE[(c ^ b) & 0xff] ^ (c >>> 8);298 return (c ^ 0xffffffff) >>> 0;299}300301function chunk(type: string, body: Uint8Array): Uint8Array {302 const out = new Uint8Array(12 + body.length);303 const view = new DataView(out.buffer);304 view.setUint32(0, body.length);305 out.set(new TextEncoder().encode(type), 4);306 out.set(body, 8);307 view.setUint32(8 + body.length, crc32(out.subarray(4, 8 + body.length)));308 return out;309}310311export function png(size: number, dark: (x: number, y: number) => boolean): Uint8Array {312 const raw = new Uint8Array(size * (size + 1));313 for (let y = 0; y < size; y++) for (let x = 0; x < size; x++) raw[y * (size + 1) + 1 + x] = dark(x, y) ? 0 : 255;314 const head = new Uint8Array(13);315 const view = new DataView(head.buffer);316 view.setUint32(0, size);317 view.setUint32(4, size);318 head[8] = 8;319 const parts = [SIGNATURE, chunk("IHDR", head), chunk("IDAT", new Uint8Array(deflateSync(raw))), chunk("IEND", new Uint8Array(0))];320 const out = new Uint8Array(parts.reduce((n, part) => n + part.length, 0));321 let at = 0;322 for (const part of parts) {323 out.set(part, at);324 at += part.length;325 }326 return out;327}328329export function icons(): Output[] {330 const rows = grid(1);331 const mark = (size: number) => png(size, (x, y) => rows[Math.floor((y * 5) / size)][Math.floor((x * 5) / size)] === "1");332 return [333 { path: "favicon.svg", bytes: logoSvg(1, "#000000", "#ffffff") },334 { path: "favicon.png", bytes: mark(40) },335 { path: "apple-touch-icon.png", bytes: mark(180) },336 { path: "icon-192.png", bytes: mark(192) },337 { path: "icon-512.png", bytes: mark(512) },338 ];339}340341/* GLOBALS */342343const clean = (root: string) => (root ?? "").replace(/\/$/, "");344345const AGENTS = ["GPTBot", "ClaudeBot", "Claude-Web", "CCBot", "Google-Extended", "anthropic-ai", "PerplexityBot"];346347function links(site: Site): Link[] {348 const out: Link[] = [];349 for (const route of site.routes) {350 if (route.hidden && !route.sitemap) continue;351 const list = route.urls ?? (route.route.endsWith("/") ? [{ route: route.route, name: route.name }] : []);352 for (const one of list) out.push({ route: one.route, name: one.name ?? one.route, at: one.at || route.at });353 }354 return out.sort((a, b) => a.route.localeCompare(b.route));355}356357function robots(site: Site, root: string): string {358 const deny = (site.config.robots?.disallow ?? []).map((path) => `Disallow: ${path}`);359 const lines: string[] = [];360 for (const agent of AGENTS) lines.push(`User-agent: ${agent}`, "Allow: /", "");361 lines.push("User-agent: *", "Allow: /", ...deny, "");362 lines.push(`Sitemap: ${root}/sitemap.xml`, "");363 return lines.join("\n");364}365366function llms(site: Site, root: string): string {367 const decl = site.config.llms ?? {};368 const known = new Set<string>();369 for (const route of site.routes) {370 known.add(route.route);371 for (const one of route.urls ?? []) known.add(one.route);372 }373 const rows = (decl.links ?? [])374 .filter((one) => known.has(one.href))375 .map((one) => `- [${one.name ?? one.href}](${root}${one.href})${one.note ? `: ${one.note}` : ""}`);376 const head = [`# ${site.config.title ?? ""}`, "", `> ${root}`, ""];377 if (decl.about) head.push(decl.about, "");378 return [...head, ...rows, ""].join("\n");379}380381export async function globals(site: Site, spec: Spec): Promise<Output[]> {382 const out: Output[] = [...((site as { copies?: Output[] }).copies ?? [])];383 const root = clean(site.config.root as string);384 const shown = links(site);385 const urls = shown.map((l) => `<url><loc>${escape(root + l.route)}</loc><lastmod>${l.at || today()}</lastmod></url>`);386 out.push({387 path: "sitemap.xml",388 bytes: `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join("\n")}\n</urlset>\n`,389 });390 out.push({ path: "robots.txt", bytes: robots(site, root) });391 out.push({ path: "llms.txt", bytes: llms(site, root) });392 if (site.config.manifest) out.push({ path: "manifest.webmanifest", bytes: JSON.stringify(site.config.manifest, null, 2) + "\n" });393 if (site.config.icons !== false) out.push(...icons());394 const wood = forest(site);395 if (wood) out.push({ path: "git/tree.json", bytes: JSON.stringify(wood), type: "application/json" });396 const pub = site.config.inputs?.public ? site.input("public") : null;397 if (pub) for (const file of walk(pub.path)) out.push({ path: file.slice(pub.path.length + 1), bytes: bytes(file) });398 if (spec.globals) out.push(...(await spec.globals(site)));399 return out;400}401402/* WRITE */403404const today = () => new Date().toISOString().slice(0, 10);405406function put(out: string, item: Output): boolean {407 const path = join(out, item.path);408 const body = typeof item.bytes === "string" ? new TextEncoder().encode(item.bytes) : item.bytes;409 if (existsSync(path)) {410 const old = new Uint8Array(readFileSync(path));411 if (old.length === body.length && Buffer.compare(old, body) === 0) return false;412 }413 mkdirSync(dirname(path), { recursive: true });414 writeFileSync(path, body);415 return true;416}417418/* BUILD */419420function typed(outputs: Output[]): Record<string, string> | undefined {421 const out: Record<string, string> = {};422 for (const item of outputs) if (item.type) out[item.path] = item.type;423 return Object.keys(out).length ? out : undefined;424}425426export async function build(spec: Spec, options: { manifest?: string; force?: boolean; verify?: boolean } = {}) {427 const site = await scan(spec);428 const path = options.manifest ? resolve(spec.root, options.manifest) : "";429 const old: Manifest = path && existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : {};430 const next: Manifest = {};431 const kept = new Set<string>();432 const verify = options.verify ?? true;433 let rendered = 0;434 let written = 0;435 for (const route of site.routes) {436 const hash = fingerprint(site, route);437 const was = old[route.route];438 const same = !options.force && !!was && was.hash === hash;439 if (was && same && (!verify || was.outputs.every((p) => existsSync(join(site.out, p))))) {440 next[route.route] = was;441 route.at = route.at || was.at;442 for (const p of was.outputs) kept.add(p);443 continue;444 }445 const outputs = await render(site, route, spec);446 for (const item of outputs) if (put(site.out, item)) written++;447 rendered++;448 const at = route.at || (was && same ? was.at : today());449 next[route.route] = { hash, at, outputs: outputs.map((o) => o.path), types: typed(outputs) };450 route.at = at;451 for (const item of outputs) kept.add(item.path);452 }453 for (const item of await globals(site, spec)) {454 if (put(site.out, item)) written++;455 kept.add(item.path);456 }457 let removed = 0;458 for (const record of Object.values(old)) {459 for (const p of record.outputs) {460 if (kept.has(p)) continue;461 const file = join(site.out, p);462 if (!existsSync(file)) continue;463 unlinkSync(file);464 removed++;465 }466 }467 if (path) {468 mkdirSync(dirname(path), { recursive: true });469 writeFileSync(path, JSON.stringify(next, null, 2) + "\n");470 }471 return { site, manifest: next, rendered, written, removed };472}473474/* HELPERS */475476export const page = (route: string) => (route === "/" ? "index.html" : `${route.replace(/^\/|\/$/g, "")}/index.html`);477478export { escape, digest, short, rmSync, today };