net.ts
8.3 kB · typescript · 232 lines
1import type { S3Client } from "bun";2import { existsSync, readdirSync, renameSync, rmSync } from "node:fs";3import { join } from "node:path";4import { client, DEV_BUCKET, getText, putBytes } from "./s3.ts";56/* WHERE */78export const SOURCE = "mrlyprod/mrlyprod";9const HEAD_KEY = "build/net/head";10const AGENT = "mrlynet-site-builder";11const SRC_DIR = "/tmp/src";12const SHELF_DIR = "/tmp/shelf";13const CACHE_DIR = process.env.BUN_INSTALL_CACHE_DIR || "/tmp/bun/cache";14const BACKOFF = [1000, 3000, 9000];1516const shelfRepo = () => (process.env.SHELF_REPO ?? "").trim();1718/* CLOCK */1920let mark = Date.now();2122function log(line: string) {23 const now = Date.now();24 console.log(`${line} ${now - mark}ms`);25 mark = now;26}2728/* RETRY */2930async function retry<T>(label: string, work: () => Promise<T>): Promise<T> {31 for (let attempt = 0; ; attempt++) {32 try {33 return await work();34 } catch (error) {35 if (attempt === BACKOFF.length) throw error;36 const wait = BACKOFF[attempt] ?? 1000;37 log(`${label} failed, retry ${attempt + 1} in ${wait}ms: ${String((error as Error).message ?? error).slice(0, 160)}`);38 await Bun.sleep(wait);39 }40 }41}4243/* EVENT */4445export type Source = "push" | "schedule" | "automator" | "manual" | "";4647export type Wake = { source: Source; on: "source" | "shelf" | ""; sha: string };4849const SOURCES = new Set(["push", "schedule", "automator", "manual"]);50const SHA = /^[0-9a-f]{7,40}$/;5152function json(text: string): Record<string, unknown> {53 try {54 const value = JSON.parse(text);55 return value && typeof value === "object" ? (value as Record<string, unknown>) : {};56 } catch {57 return {};58 }59}6061export function readEvent(text: string): Wake {62 const outer = json(text);63 const body = typeof outer.body === "string" ? json(outer.body) : outer;64 const word = typeof body.source === "string" ? body.source.trim() : "";65 const repo = typeof body.repo === "string" ? body.repo.trim() : "";66 const named = typeof body.sha === "string" ? body.sha.trim().toLowerCase() : "";67 return {68 source: SOURCES.has(word) ? (word as Source) : "",69 on: repo === SOURCE ? "source" : (repo && repo === shelfRepo() ? "shelf" : ""),70 sha: SHA.test(named) ? named : "",71 };72}7374async function payload(request?: Request): Promise<string> {75 if (!request) return "";76 try {77 return await request.text();78 } catch {79 return "";80 }81}8283/* HEAD */8485type Head = { sha: string; etag: string; shelf: string; shelfEtag: string };8687type Mark = { sha: string; etag: string };8889const short = (sha: string) => sha.slice(0, 7) || "none";9091async function readHead(s3: S3Client): Promise<Head> {92 const text = await getText(s3, HEAD_KEY);93 const [sha = "", etag = "", shelf = "", shelfEtag = ""] = (text ?? "").trim().split("\n");94 return { sha: sha.trim(), etag: etag.trim(), shelf: shelf.trim(), shelfEtag: shelfEtag.trim() };95}9697async function writeHead(s3: S3Client, head: Head): Promise<void> {98 const body = `${head.sha}\n${head.etag}\n${head.shelf}\n${head.shelfEtag}\n`;99 await putBytes(s3, HEAD_KEY, body, { type: "text/plain", cacheControl: "no-store" });100}101102/* GITHUB */103104async function commit(slug: string, etag: string): Promise<Mark | null> {105 return retry(`github ${slug}`, async () => {106 const headers: Record<string, string> = { "user-agent": AGENT, accept: "application/vnd.github+json" };107 if (etag) headers["if-none-match"] = etag;108 const res = await fetch(`https://api.github.com/repos/${slug}/commits/main`, { headers });109 if (res.status === 304) return null;110 if (!res.ok) throw new Error(`github ${slug} ${res.status}: ${(await res.text()).slice(0, 200)}`);111 const body = (await res.json()) as { sha?: string };112 if (!body.sha) throw new Error(`github ${slug}: the commit carries no sha`);113 return { sha: body.sha, etag: res.headers.get("etag") ?? "" };114 });115}116117async function unpack(slug: string, ref: string, into: string): Promise<void> {118 await retry(`codeload ${slug} ${short(ref)}`, async () => {119 const stage = `${into}.stage`;120 rmSync(stage, { recursive: true, force: true });121 rmSync(into, { recursive: true, force: true });122 const res = await fetch(`https://codeload.github.com/${slug}/tar.gz/${ref}`, { headers: { "user-agent": AGENT } });123 if (!res.ok) throw new Error(`codeload ${slug} ${ref}: ${res.status}`);124 const files = await new Bun.Archive(new Uint8Array(await res.arrayBuffer())).extract(stage);125 const [top] = readdirSync(stage);126 if (!files || !top) throw new Error(`codeload ${slug} ${ref}: the tarball is empty`);127 renameSync(join(stage, top), into);128 rmSync(stage, { recursive: true, force: true });129 });130}131132/* PLAN */133134function complete(stored: Head): boolean {135 return Boolean(stored.sha) && (!shelfRepo() || Boolean(stored.shelf));136}137138function seen(wake: Wake, stored: Head): string {139 if (!wake.sha || !complete(stored)) return "";140 if (wake.on === "source" && wake.sha === stored.sha) return short(wake.sha);141 if (wake.on === "shelf" && wake.sha === stored.shelf) return `shelf ${short(wake.sha)}`;142 return "";143}144145async function shelfMark(wake: Wake, stored: Head): Promise<Mark> {146 const slug = shelfRepo();147 if (!slug) return { sha: "", etag: "" };148 if (wake.on === "shelf" && wake.sha) return { sha: wake.sha, etag: "" };149 return (await commit(slug, stored.shelf ? stored.shelfEtag : "")) ?? { sha: stored.shelf, etag: stored.shelfEtag };150}151152async function freshen(wake: Wake, stored: Head): Promise<Head> {153 const source =154 wake.on === "source" && wake.sha155 ? { sha: wake.sha, etag: "" }156 : ((await commit(SOURCE, stored.sha ? stored.etag : "")) ?? { sha: stored.sha, etag: stored.etag });157 const shelf = await shelfMark(wake, stored);158 return { sha: source.sha, etag: source.etag, shelf: shelf.sha, shelfEtag: shelf.etag };159}160161/* CHILD */162163function childEnv(): Record<string, string> {164 const env: Record<string, string> = {165 ...(process.env as Record<string, string>),166 HOME: "/tmp",167 BUN_INSTALL_CACHE_DIR: CACHE_DIR,168 };169 if (shelfRepo()) env.MRLY_SHELF = join(SHELF_DIR, "research");170 else delete env.MRLY_SHELF;171 return env;172}173174async function run(cmd: string[], cwd: string): Promise<string> {175 const child = Bun.spawn(cmd, { cwd, env: childEnv(), stdout: "pipe", stderr: "pipe" });176 const [out, err] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text()]);177 const code = await child.exited;178 if (code !== 0) throw new Error(`${cmd.join(" ")}: exit ${code}\n${(err || out).trim().slice(-1500)}`);179 return out;180}181182/* BUILD */183184async function build(s3: S3Client, next: Head): Promise<string> {185 await unpack(SOURCE, next.sha, SRC_DIR);186 log(`source ${short(next.sha)}`);187 const slug = shelfRepo();188 if (slug) {189 await unpack(slug, next.shelf || "main", SHELF_DIR);190 if (!existsSync(join(SHELF_DIR, "research", "README.md"))) throw new Error("shelf: no research/README.md in the tarball");191 log(`shelf ${short(next.shelf)}`);192 }193 const site = join(SRC_DIR, "site");194 await run([process.execPath, "install", "--frozen-lockfile"], site);195 log("install");196 const pkg = await run([process.execPath, "scripts/pkg.ts"], site);197 log(`pkg ${(pkg.trim().split(/\s+/)[0] ?? "").slice(0, 12)}`);198 const out = await run([process.execPath, "run", "push"], site);199 const line = out.split("\n").map((one) => one.trim()).find((one) => one.startsWith("push:")) ?? "push: no count line";200 log(line);201 await writeHead(s3, next);202 log(`head ${short(next.sha)} ${short(next.shelf)}`);203 return line;204}205206/* HANDLER */207208async function handler(request?: Request): Promise<Response> {209 const began = Date.now();210 mark = began;211 const wake = readEvent(await payload(request));212 const s3 = client(DEV_BUCKET);213 const stored = await readHead(s3);214 const known = seen(wake, stored);215 if (known) {216 log(`seen ${known}`);217 return new Response(`seen ${known}\n`);218 }219 const next = await freshen(wake, stored);220 if (next.sha === stored.sha && next.shelf === stored.shelf) {221 if (next.etag !== stored.etag || next.shelfEtag !== stored.shelfEtag) await writeHead(s3, next);222 log(`unchanged ${short(next.sha)}`);223 return new Response(`unchanged ${short(next.sha)}\n`);224 }225 log(`${wake.source || "poll"} ${short(next.sha)} shelf ${short(next.shelf)}`);226 const line = await build(s3, next);227 const total = Date.now() - began;228 console.log(`done ${short(next.sha)} ${total}ms`);229 return new Response(`${line} in ${total}ms\n`);230}231232export default { fetch: handler };