press.ts

1import { createHash } from "node:crypto";2import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";3import { availableParallelism } from "node:os";4import { dirname, join, relative, resolve } from "node:path";5import sharp from "sharp";6import { ink, raster, svg, type Ink, type Palette, type Pen, type Pixels } from "mrlyjs/view";78/* WHERE */910const USAGE = "usage: bun press.ts [name ...] [--svg] [--png]";11const HERE = import.meta.dir;12const DESK = resolve(HERE, "../..");13const MRLYJS = resolve(HERE, "../pkgs/mrlyjs");14const DATA_DIR = join(DESK, "data", relative(DESK, HERE));15const STORE = join(DATA_DIR, "store");16const LOCK = join(DATA_DIR, "figures.lock");17const BENCH = join(DATA_DIR, "press.txt");1819/* RECIPE */2021const FIGURE = /^(site|research|paper|wiki|demo|blog)-[a-z0-9-]+\.ts$/;22const KIT = /^(pkg\/.+\.(js|wasm)|(view\/)?[^/]+\.js)$/;23const THEMES = ["dark", "light"] as const;24const SIZE = [1024, 1024];25const BUDGET = 5000;26const WIDTH = 1024;27const WEBP = { quality: 90, effort: 6 };28const PNG = { compressionLevel: 9 };2930type Theme = (typeof THEMES)[number];31type Flags = { svg: boolean; png: boolean };32type Job = { name: string; file: string; dir: string; missing: string[]; palettes: Record<Theme, Palette> };33type Row = { state: "drawn" | "cached" | "failed"; name: string; theme: string; draw?: number; encode?: number; extra?: number; note?: string };34type Done = { rows: Row[]; ms: number };35type Unit = { initSync: (options: { module: Uint8Array }) => unknown };36type Figure = { default: (pen: Pen, ink: Ink, t: number) => void; size?: number[]; still?: number; units?: Record<string, Unit> };3738const reason = (error: unknown) => (error instanceof Error ? error.message : String(error)).split("\n")[0];3940/* KEY */4142const sha = (data: string | Uint8Array) => createHash("sha256").update(data).digest("hex");43const scanner = new Bun.Transpiler({ loader: "ts" });4445function digest(files: string[], root: string) {46  const lines = files.map((file) => `${sha(readFileSync(file))}  ${relative(root, file)}\n`);47  return sha(lines.sort().join(""));48}4950function kit() {51  const files = (readdirSync(MRLYJS, { recursive: true }) as string[]).filter((file) => KIT.test(file) && !file.endsWith(".test.js"));52  return digest(files.map((file) => join(MRLYJS, file)), MRLYJS);53}5455function sources(file: string, seen = new Set<string>()) {56  if (seen.has(file)) return seen;57  seen.add(file);58  if (!/\.[cm]?[jt]s$/.test(file)) return seen;59  for (const { path } of scanner.scanImports(readFileSync(file, "utf8"))) {60    if (path.startsWith(".")) sources(Bun.resolveSync(path, dirname(file)), seen);61  }62  return seen;63}6465const key = (base: string, name: string, file: string) => sha(`${base}\n${name}\n${digest([...sources(file)], HERE)}`);6667/* WORKER */6869const pixels = (p: Pixels) => sharp(p.colors, { raw: { width: p.shape[1], height: p.shape[0], channels: 4 } });7071async function clock(work: () => Promise<void>) {72  const start = performance.now();73  await work();74  return performance.now() - start;75}7677async function press(job: Job, ready: Set<string>) {78  const rows: Row[] = [];79  let theme = "-";80  try {81    const figure: Figure = await import(job.file);82    if (typeof figure.default !== "function") throw new Error("no default draw");83    for (const [unit, door] of Object.entries(figure.units ?? {})) {84      if (ready.has(unit)) continue;85      door.initSync({ module: readFileSync(join(MRLYJS, "pkg", unit, `mrlyjs_${unit}_bg.wasm`)) });86      ready.add(unit);87    }88    const [width, height] = figure.size ?? SIZE;89    const missing = new Set(job.missing);90    mkdirSync(job.dir, { recursive: true });91    const write = (file: string, bytes: Uint8Array | string) => {92      writeFileSync(join(job.dir, `${file}.tmp`), bytes);93      renameSync(join(job.dir, `${file}.tmp`), join(job.dir, file));94    };95    const draw = (pen: Pen, tint: Ink) => {96      postMessage("draw");97      const start = performance.now();98      figure.default(pen, tint, figure.still ?? 0);99      const ms = performance.now() - start;100      postMessage("drawn");101      if (ms > BUDGET) throw new Error(`draw took ${ms.toFixed(0)} ms, over ${BUDGET}`);102      return ms;103    };104    for (theme of THEMES) {105      const tint = ink(job.palettes[theme as Theme]);106      const row = { state: "drawn" as const, name: job.name, theme, draw: undefined as number | undefined, encode: 0, extra: 0 };107      const webp = `${job.name}-${theme}.webp`;108      const png = `${job.name}-${theme}.png`;109      const vector = `${job.name}-${theme}.svg`;110      const og = theme === "dark" && missing.has(`${job.name}.png`);111      if (missing.has(webp) || missing.has(png) || og) {112        const pen = raster(width, height, tint.ground);113        row.draw = draw(pen, tint);114        if (missing.has(webp)) {115          row.encode += await clock(async () => {116            const image = pixels(pen.pixels());117            write(webp, await (width > WIDTH ? image.resize({ width: WIDTH, kernel: "lanczos3" }) : image).webp(WEBP).toBuffer());118          });119        }120        if (og || missing.has(png)) {121          row[og ? "encode" : "extra"] += await clock(async () => {122            const bytes = await pixels(pen.pixels()).png(PNG).toBuffer();123            if (og) write(`${job.name}.png`, bytes);124            if (missing.has(png)) write(png, bytes);125          });126        }127      }128      if (missing.has(vector)) {129        row.extra += await clock(async () => {130          const pen = svg(width, height, tint.ground);131          draw(pen, tint);132          write(vector, await pen.text({ png: (p) => pixels(p).png().toBuffer() }));133        });134      }135      rows.push(row);136    }137  } catch (error) {138    rows.push({ state: "failed", name: job.name, theme, note: reason(error) });139  }140  return rows;141}142143function serve() {144  const ready = new Set<string>();145  self.onmessage = async ({ data }: MessageEvent<Job>) => {146    const start = performance.now();147    const rows = await press(data, ready);148    postMessage({ rows, ms: performance.now() - start });149  };150}151152/* CREW */153154function run(worker: Worker, job: Job) {155  return new Promise<Done | string>((done) => {156    let timer: ReturnType<typeof setTimeout>;157    const arm = () => {158      clearTimeout(timer);159      timer = setTimeout(() => done(`over ${BUDGET} ms`), BUDGET);160    };161    worker.onmessage = ({ data }: MessageEvent<"draw" | "drawn" | Done>) => {162      if (data === "draw") return arm();163      clearTimeout(timer);164      if (data !== "drawn") done(data);165    };166    worker.onerror = (event) => {167      clearTimeout(timer);168      done(reason(event.message || "the worker died"));169    };170    arm();171    worker.postMessage(job);172  });173}174175function crew(jobs: Job[], cores: number, note: (done: Done) => void) {176  let next = 0;177  const lane = async () => {178    let worker = new Worker(import.meta.url);179    while (next < jobs.length) {180      const job = jobs[next++];181      const done = await run(worker, job);182      if (typeof done !== "string") note(done);183      else {184        worker.terminate();185        worker = new Worker(import.meta.url);186        note({ rows: [{ state: "failed", name: job.name, theme: "-", note: done }], ms: 0 });187      }188    }189    worker.terminate();190  };191  return Promise.all(Array.from({ length: Math.min(cores, jobs.length) }, lane));192}193194/* MAIN */195196function args(argv: string[]) {197  const flags: Flags = { svg: false, png: false };198  const names = new Set<string>();199  for (const arg of argv) {200    if (arg === "--svg") flags.svg = true;201    else if (arg === "--png") flags.png = true;202    else if (arg.startsWith("-")) throw new Error(USAGE);203    else names.add(arg);204  }205  return { flags, names: [...names] };206}207208function outputs(name: string, flags: Flags) {209  const out = [`${name}-dark.webp`, `${name}-light.webp`, `${name}.png`];210  if (flags.svg) out.push(`${name}-dark.svg`, `${name}-light.svg`);211  if (flags.png) out.push(`${name}-dark.png`, `${name}-light.png`);212  return out;213}214215const ms = (value?: number) => (value === undefined ? "-" : value.toFixed(1)).padStart(8);216217function line(row: Row) {218  const extra = row.extra ? `  extra ${row.extra.toFixed(1)}` : "";219  return `${row.state.padEnd(6)}  ${ms(row.draw)}  ${ms(row.encode)}  ${row.name}  ${row.theme}${extra}${row.note ? `  ${row.note}` : ""}`;220}221222async function main() {223  const { flags, names } = args(process.argv.slice(2));224  const cores = availableParallelism();225  const all = readdirSync(HERE).filter((file) => FIGURE.test(file)).map((file) => file.slice(0, -3)).sort();226  const core = await import("mrlyjs/core");227  core.initSync({ module: readFileSync(join(MRLYJS, "pkg/core/mrlyjs_core_bg.wasm")) });228  const palettes: Record<Theme, Palette> = { dark: core.colors.DARK().toJSON(), light: core.colors.LIGHT().toJSON() };229  const base = sha(JSON.stringify([sha(readFileSync(import.meta.path)), kit(), palettes]));230  const rows: Row[] = [];231  const keys: Record<string, string> = {};232  const jobs: Job[] = [];233  let work = 0;234  const note = (row: Row) => {235    rows.push(row);236    console.log(line(row));237  };238  const wanted = names.length ? names : all;239  for (const name of wanted) {240    try {241      if (!all.includes(name)) throw new Error("no such figure");242      const file = join(HERE, `${name}.ts`);243      keys[name] = key(base, name, file);244      const dir = join(STORE, keys[name]);245      const missing = outputs(name, flags).filter((out) => !existsSync(join(dir, out)));246      if (missing.length) jobs.push({ name, file, dir, missing, palettes });247      else for (const theme of THEMES) note({ state: "cached", name, theme });248    } catch (error) {249      note({ state: "failed", name, theme: "-", note: reason(error) });250    }251  }252  await crew(jobs, cores, (done) => {253    work += done.ms;254    done.rows.forEach(note);255  });256  const named = (state: Row["state"]) => new Set(rows.filter((row) => row.state === state).map((row) => row.name));257  const failed = named("failed");258  const drawn = [...named("drawn")].filter((name) => !failed.has(name));259  const lock: Record<string, string> = names.length && existsSync(LOCK) ? JSON.parse(readFileSync(LOCK, "utf8")) : {};260  for (const [name, hash] of Object.entries(keys)) if (!failed.has(name)) lock[name] = hash;261  const held = new Set(Object.values(lock));262  if (!names.length && existsSync(STORE)) {263    for (const stale of readdirSync(STORE)) if (!held.has(stale)) rmSync(join(STORE, stale), { recursive: true });264  }265  const last = `${wanted.length} figures: ${drawn.length} drawn, ${named("cached").size} cached, ${failed.size} failed, wall ${performance.now().toFixed(0)} ms, work ${work.toFixed(0)} ms, ${cores} cores`;266  console.log(last);267  mkdirSync(DATA_DIR, { recursive: true });268  writeFileSync(LOCK, `${JSON.stringify(Object.fromEntries(Object.entries(lock).sort()), null, 2)}\n`);269  const sorted = [...rows].sort((a, b) => (a.name + a.theme < b.name + b.theme ? -1 : 1));270  writeFileSync(BENCH, `${[...sorted.map(line), last].join("\n")}\n`);271  process.exit(failed.size ? 1 : 0);272}273274if (!Bun.isMainThread) serve();275else if (import.meta.main) await main();