shelf.ts

1import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";2import { dirname, join, resolve } from "node:path";34const org = resolve(import.meta.dir, "..");5const data = join(org, "data");6const home = join(data, "shelf");7const REPO = process.env.SHELF_REPO ?? "";8const TARBALL = REPO ? `https://codeload.github.com/${REPO}/tar.gz/main` : "";9const KEEP = "research/";1011/* TAR */1213const text = (bytes: Uint8Array, from: number, to: number) => {14  const end = bytes.indexOf(0, from);15  return new TextDecoder().decode(bytes.subarray(from, end < 0 || end > to ? to : end));16};17const octal = (bytes: Uint8Array, from: number, to: number) => parseInt(text(bytes, from, to).trim() || "0", 8);1819function pax(block: Uint8Array) {20  const out: Record<string, string> = {};21  const body = new TextDecoder().decode(block);22  for (const line of body.split("\n")) {23    const m = line.match(/^\d+ ([^=]+)=(.*)$/);24    if (m) out[m[1]] = m[2];25  }26  return out;27}2829function untar(tar: Uint8Array, into: string) {30  let at = 0;31  let next: Record<string, string> = {};32  let files = 0;33  while (at + 512 <= tar.length) {34    const head = tar.subarray(at, at + 512);35    if (head.every((b) => b === 0)) break;36    const size = octal(head, 124, 136);37    const kind = String.fromCharCode(head[156]);38    const prefix = text(head, 345, 500);39    let name = next.path ?? (prefix ? `${prefix}/${text(head, 0, 100)}` : text(head, 0, 100));40    const body = tar.subarray(at + 512, at + 512 + size);41    at += 512 + Math.ceil(size / 512) * 512;42    if (kind === "x") {43      next = pax(body);44      continue;45    }46    next = {};47    if (kind === "g" || kind === "L") continue;48    name = name.replace(/^[^/]+\//, "");49    if (!name.startsWith(KEEP)) continue;50    const path = join(into, name);51    if (kind === "5" || name.endsWith("/")) {52      mkdirSync(path, { recursive: true });53    } else if (kind === "0" || kind === "\0") {54      mkdirSync(dirname(path), { recursive: true });55      writeFileSync(path, body);56      files++;57    }58  }59  return files;60}6162/* FETCH */6364export async function shelf(): Promise<string> {65  const local = process.env.MRLY_SHELF;66  if (local) return resolve(local);67  const cached = join(home, "research");68  const fallback = (why: string) => {69    if (!existsSync(join(cached, "README.md"))) throw new Error(`shelf: ${why} and no cache at ${cached}; set MRLY_SHELF to a local checkout or SHELF_REPO to an owner/repo`);70    console.warn(`shelf: ${why}, building from the cached copy`);71    return cached;72  };73  if (!TARBALL) return fallback("no SHELF_REPO in the environment");74  try {75    const res = await fetch(TARBALL);76    if (!res.ok) throw new Error(`HTTP ${res.status}`);77    const tar = Bun.gunzipSync(new Uint8Array(await res.arrayBuffer()));78    const fresh = join(data, "shelf.next");79    rmSync(fresh, { recursive: true, force: true });80    const files = untar(tar, fresh);81    if (!existsSync(join(fresh, "research", "README.md"))) throw new Error("tarball carries no research/README.md");82    rmSync(home, { recursive: true, force: true });83    renameSync(fresh, home);84    console.log(`shelf: fetched ${files} files from GitHub`);85  } catch (reason) {86    return fallback(`fetch failed (${reason})`);87  }88  return cached;89}