push.ts

1import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";2import { tmpdir } from "node:os";3import { dirname, join } from "node:path";4import { build, digest, globals, today, type Manifest, type Output, type Spec } from "./ssg/build.ts";5import { client, del, getText, need, putBytes } from "./s3.ts";67/* WHERE */89export type Store = { bucket: string; prefix: string };1011export type Block = { prefix: string; guard: string[]; bucket: string; store: Store; hashed: string[] };1213const ASSET = "@";14const BATCH = 32;1516export function block(spec: Spec): Block {17  const config = spec.config ?? (JSON.parse(readFileSync(join(spec.root, "site.json"), "utf8")) as Record<string, unknown>);18  const found = config.push as Block | undefined;19  if (!found) throw new Error("push: site.json has no push block");20  return found;21}2223/* HEADERS */2425export const IMMUTABLE = "public, max-age=31536000, immutable";26export const REVALIDATE = "public, max-age=0, must-revalidate";2728const TYPES: Record<string, string> = {29  html: "text/html; charset=utf-8",30  js: "text/javascript; charset=utf-8",31  mjs: "text/javascript; charset=utf-8",32  css: "text/css; charset=utf-8",33  json: "application/json",34  map: "application/json",35  webmanifest: "application/manifest+json",36  xml: "application/xml",37  txt: "text/plain; charset=utf-8",38  md: "text/markdown; charset=utf-8",39  tex: "text/plain; charset=utf-8",40  wasm: "application/wasm",41  svg: "image/svg+xml",42  png: "image/png",43  jpg: "image/jpeg",44  jpeg: "image/jpeg",45  gif: "image/gif",46  webp: "image/webp",47  ico: "image/x-icon",48  pdf: "application/pdf",49  woff2: "font/woff2",50};5152export const kind = (path: string) => TYPES[(path.match(/\.([^./]+)$/)?.[1] ?? "").toLowerCase()] ?? "application/octet-stream";5354export const cache = (path: string, hashed: RegExp[]) => (hashed.some((re) => re.test(path)) ? IMMUTABLE : REVALIDATE);5556export const rules = (hashed: string[]) => hashed.map((one) => new RegExp(one));5758/* GUARD */5960export const mine = (path: string, guard: string[]) => !guard.some((one) => path.startsWith(one));6162type Page = {63  contents?: { key: string }[];64  commonPrefixes?: ({ prefix: string } | string)[];65  isTruncated?: boolean;66  nextContinuationToken?: string | null;67};6869export type Lister = { list: (options: { prefix: string; delimiter: string; maxKeys: number; continuationToken?: string }) => Promise<Page | null> };7071export async function sweep(s3: Lister, root: string, guard: string[], at = root, out: string[] = []): Promise<string[]> {72  let token: string | undefined;73  do {74    const page = await s3.list({ prefix: at, delimiter: "/", maxKeys: 1000, continuationToken: token });75    for (const item of page?.contents ?? []) out.push(item.key.slice(root.length));76    for (const one of page?.commonPrefixes ?? []) {77      const next = typeof one === "string" ? one : one.prefix;78      if (!guard.some((each) => next === root + each)) await sweep(s3, root, guard, next, out);79    }80    token = page?.isTruncated ? (page.nextContinuationToken ?? undefined) : undefined;81  } while (token);82  return out;83}8485/* MANIFEST */8687export function assets(items: Output[], old: Manifest): Manifest {88  const out: Manifest = {};89  for (const item of items) {90    const body = typeof item.bytes === "string" ? new TextEncoder().encode(item.bytes) : item.bytes;91    const hash = digest([body]).slice(0, 16);92    const was = old[ASSET + item.path];93    out[ASSET + item.path] = was && was.hash === hash ? was : { hash, at: today(), outputs: [item.path] };94  }95  return out;96}9798export function typing(manifest: Manifest): Map<string, string> {99  const out = new Map<string, string>();100  for (const record of Object.values(manifest)) for (const [path, type] of Object.entries(record.types ?? {})) out.set(path, type);101  return out;102}103104export function spread(manifest: Manifest): Map<string, string> {105  const out = new Map<string, string>();106  for (const [key, record] of Object.entries(manifest)) for (const path of record.outputs) out.set(path, key);107  return out;108}109110/* LOCAL */111112export const holding = () => process.env.DRY === "1";113114const where = (store: Store) => join(process.env.DRY_DIR ?? join(tmpdir(), "push", store.prefix), "manifest.json");115116const held = (path: string) => (existsSync(path) ? readFileSync(path, "utf8") : null);117118function hold(path: string, text: string) {119  mkdirSync(dirname(path), { recursive: true });120  writeFileSync(path, text);121}122123/* PUSH */124125export async function push(spec: Spec, options: { dry?: boolean } = {}): Promise<{ rendered: number; uploaded: number; deleted: number }> {126  const conf = block(spec);127  const hashed = rules(conf.hashed);128  const local = holding();129  const site = local ? null : client(need(conf.bucket));130  const store = local ? null : conf.store.bucket === conf.bucket ? site : client(need(conf.store.bucket));131  const key = `${conf.store.prefix}/manifest.json`;132  const found = store ? await getText(store, key) : held(where(conf.store));133  const old: Manifest = found ? JSON.parse(found) : {};134  const carry = join(tmpdir(), `push-remote-${process.pid}.json`);135  writeFileSync(carry, JSON.stringify(old, null, 2) + "\n");136  const done = await build(spec, { manifest: carry, verify: false });137  rmSync(carry, { force: true });138  const next: Manifest = { ...done.manifest, ...assets(await globals(done.site, spec), old) };139  const want = spread(next);140  const had = spread(old);141  const types = typing(next);142  const upload: string[] = [];143  for (const [path, name] of want) {144    if (!mine(path, conf.guard)) continue;145    const was = old[name];146    if (was && was.hash === next[name]!.hash && had.has(path)) continue;147    upload.push(path);148  }149  const seen = found || !site ? [...had.keys()] : await sweep(site, conf.prefix, conf.guard);150  const remove = seen.filter((path) => mine(path, conf.guard) && !want.has(path));151  const header = (path: string) => (types.has(path) ? REVALIDATE : cache(path, hashed));152  const text = JSON.stringify(next, null, 2) + "\n";153  if (options.dry) {154    const fixed = upload.filter((path) => header(path) === IMMUTABLE);155    for (const path of fixed) console.log(`immutable ${conf.prefix + path}`);156    console.log(`${upload.length - fixed.length} more at max-age 0, ${remove.length} to delete`);157  } else if (site && store) {158    for (let i = 0; i < upload.length; i += BATCH) {159      await Promise.all(160        upload.slice(i, i + BATCH).map((path) =>161          putBytes(site, conf.prefix + path, new Uint8Array(readFileSync(join(done.site.out, path))), {162            type: types.get(path) ?? kind(path),163            cacheControl: header(path),164          }),165        ),166      );167    }168    if (remove.length) await del(site, remove.map((path) => conf.prefix + path), BATCH);169    await putBytes(store, key, text, { type: "application/json", cacheControl: REVALIDATE });170  } else hold(where(conf.store), text);171  return { rendered: done.rendered, uploaded: upload.length, deleted: remove.length };172}173174/* MAIN */175176export async function main(spec: Spec): Promise<void> {177  const dry = process.argv.includes("--dry");178  const done = await push(spec, { dry });179  const guard = block(spec).guard.join(", ");180  console.log(181    `push${dry ? " --dry" : ""}${holding() ? " --local" : ""}: ${done.rendered} rendered, ${done.uploaded} uploaded, ${done.deleted} deleted, ${guard} guarded`,182  );183}