check.ts

1import { existsSync, readdirSync, readFileSync } from 'node:fs';2import { dirname, join, resolve } from 'node:path';34const started = performance.now();5const root = resolve(import.meta.dir, '..');6const at = (...parts: string[]) => join(root, ...parts);7const there = (...parts: string[]) => existsSync(at(...parts));8const checks: [string, unknown, unknown][] = [];9const report = (label: string, tally: string, bad: string[]) =>10  checks.push([label, bad.length ? `${bad.length} bad: ${bad.slice(0, 3).join('; ')}` : tally, tally]);1112// TREE1314type Doc = { name: string; lines: string[]; front: Map<string, string>; body: number };1516const desks = new Map<string, Doc>();1718const read = (name: string): Doc => {19  const held = desks.get(name);20  if (held) return held;21  const lines = readFileSync(at(name), 'utf8').split('\n');22  const front = new Map<string, string>();23  let body = 0;24  if (lines[0] === '---') {25    const close = lines.indexOf('---', 1);26    if (close > 0) {27      for (const line of lines.slice(1, close)) {28        const cut = line.indexOf(':');29        if (cut > 0) front.set(line.slice(0, cut).trim(), line.slice(cut + 1).trim());30      }31      body = close + 1;32    }33  }34  const doc = { name, lines, front, body };35  desks.set(name, doc);36  return doc;37};3839const sheets = (folder: string) =>40  there(folder) ? readdirSync(at(folder)).filter((n) => n.endsWith('.md')).sort().map((n) => `${folder}/${n}`) : [];4142const under = (folder: string, suffix: string) => {43  const out: string[] = [];44  const walk = (here: string) => {45    for (const entry of readdirSync(at(here), { withFileTypes: true })) {46      if (entry.isDirectory()) walk(`${here}/${entry.name}`);47      else if (entry.name.endsWith(suffix)) out.push(`${here}/${entry.name}`);48    }49  };50  if (there(folder)) walk(folder);51  return out;52};5354const stem = (name: string) => name.slice(name.lastIndexOf('/') + 1).replace(/\.[a-z]+$/, '');5556const plain = (doc: Doc) => {57  const out: [number, string][] = [];58  let fence = false;59  doc.lines.forEach((line, i) => {60    if (/^\s*(```|~~~)/.test(line)) { fence = !fence; return; }61    if (!fence) out.push([i + 1, line]);62  });63  return out;64};6566const notes = sheets('research/notes').map(read);67const claims = sheets('research/claims').map(read);68const papers = sheets('research/papers').map(read);69const pages = sheets('site/pages').map(read);70const posts = sheets('site/blog').map(read);71const concepts = sheets('wiki').map(read);72const stems = ['research/README.md', 'research/REFS.md', 'research/sequences.md'].filter((name) => there(name)).map(read);73const fronted = [...notes, ...papers, ...pages, ...posts, ...concepts];74const prose = [...notes, ...claims, ...papers, ...stems, ...pages, ...posts, ...concepts];75const housed = [...under('research', '.md'), ...sheets('site/pages'), ...sheets('site/blog'), ...sheets('wiki')].map(read);76const demos = new Set(77  there('site/demos')78    ? readdirSync(at('site/demos'), { withFileTypes: true })79        .filter((entry) => entry.isDirectory() && there(`site/demos/${entry.name}/index.html`))80        .map((entry) => entry.name)81    : [],82);83const figures = new Set(there('files/figures') ? readdirSync(at('files/figures')) : []);84const pairs = [...figures].filter((name) => name.endsWith('-dark.png')).map((name) => name.slice(0, -9));85const pair = (name: string) => figures.has(`${name}-dark.png`) && figures.has(`${name}-light.png`);86const noted = new Set(notes.map((doc) => stem(doc.name)));8788// CLAIMS8990const CLAIM = /^- (\d{4})-(\d{2})-(\d{2}) \[(Proved|Verified|Conjecture|Refuted)\] \S/;9192const real = (y: number, mo: number, d: number) => {93  const when = new Date(Date.UTC(y, mo - 1, d));94  return when.getUTCFullYear() === y && when.getUTCMonth() === mo - 1 && when.getUTCDate() === d;95};9697const rows: [Doc, number, string][] = [];98const ledger: string[] = [];99for (const doc of claims) {100  if (!doc.lines[0].startsWith('# ')) ledger.push(`${doc.name}:1 no title`);101  let last = '';102  doc.lines.forEach((line, i) => {103    const where = `${doc.name}:${i + 1}`;104    if (i === 0 || line === '') return;105    const hit = CLAIM.exec(line);106    if (!hit) { ledger.push(`${where} ${line.startsWith('- ') ? 'untagged claim' : 'stray line'}`); return; }107    rows.push([doc, i + 1, line]);108    if (!real(Number(hit[1]), Number(hit[2]), Number(hit[3]))) ledger.push(`${where} not a date`);109    const date = `${hit[1]}-${hit[2]}-${hit[3]}`;110    if (last && date < last) ledger.push(`${where} ${date} under ${last}`);111    last = date;112  });113}114report('claims', `${claims.length} files, ${rows.length} claims`, ledger);115116// POINTERS117118const NAME = /\b(?:fn|const|static|struct|enum|trait|type|mod|union)\s+([A-Za-z_]\w*)/g;119const declared = new Map<string, Set<string>>();120for (const crate of there('crates')121  ? readdirSync(at('crates'), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name)122  : []) {123  const names = new Set<string>();124  for (const file of under(`crates/${crate}/src`, '.rs')) {125    names.add(stem(file));126    for (const hit of readFileSync(at(file), 'utf8').matchAll(NAME)) names.add(hit[1]);127  }128  declared.set(crate, names);129}130131const studies = new Set(132  ['rs', 'py'].flatMap((kind) =>133    there(`research/lab/${kind}`)134      ? readdirSync(at(`research/lab/${kind}`), { withFileTypes: true })135          .filter((entry) => entry.isDirectory())136          .map((entry) => `${kind}/${entry.name}`)137      : [],138  ),139);140const sequences = new Set(141  there('research/sequences.md')142    ? [...readFileSync(at('research/sequences.md'), 'utf8').matchAll(/\bsequence_[a-z0-9_=]+/g)].map((hit) => hit[0])143    : [],144);145146const pointers: string[] = [];147let aimed = 0;148for (const [doc, n, line] of rows) {149  const blame = (what: string) => pointers.push(`${doc.name}:${n} ${what}`);150  for (const hit of line.matchAll(/\bmrly[a-z]+(?:::[A-Za-z_]\w*)+/g)) {151    aimed += 1;152    const parts = hit[0].split('::');153    const names = declared.get(parts[0]);154    if (!names) blame(`no crate ${parts[0]}`);155    else if (!names.has(parts[parts.length - 1])) blame(`${hit[0]} is undeclared`);156  }157  for (const hit of line.matchAll(/\blab\/([a-z0-9-]+)(?:\/([a-z0-9-]+))?/g)) {158    aimed += 1;159    if (hit[1] !== 'rs' && hit[1] !== 'py') blame(`lab/${hit[1]} names no kind`);160    else if (!hit[2]) blame(`lab/${hit[1]} names no study`);161    else if (!studies.has(`${hit[1]}/${hit[2]}`)) blame(`no study lab/${hit[1]}/${hit[2]}`);162  }163  for (const hit of line.matchAll(/\bsequence_[a-z0-9_=]+/g)) {164    aimed += 1;165    if (!sequences.has(hit[0])) blame(`${hit[0]} is off the ledger`);166  }167  for (const hit of line.matchAll(/\bA\d{4,8}\b/g)) {168    aimed += 1;169    if (hit[0].length !== 7) blame(`${hit[0]} is not an OEIS id`);170  }171  for (const hit of line.matchAll(/\b([a-z0-9-]+)\.md\b/g)) {172    aimed += 1;173    if (!['research/notes', 'research', 'research/claims', 'research/papers'].some((folder) => there(`${folder}/${hit[1]}.md`)))174      blame(`${hit[1]}.md names no page`);175  }176}177report('pointers', `${aimed} pointers`, pointers);178179// FIGURES180181const drawn: string[] = [];182for (const doc of fronted) {183  const figure = doc.front.get('figure');184  if (figure && !pair(figure)) drawn.push(`${doc.name} figure ${figure} has no pair`);185  for (const [n, line] of plain(doc))186    for (const hit of line.matchAll(/!\[[^\]\n]*\]\(([^)\s]+)\)/g)) {187      const target = hit[1];188      if (target.includes('/') || target.includes('.')) continue;189      if (!figures.has(`${target}.png`) && !pair(target)) drawn.push(`${doc.name}:${n} image ${target} is not drawn`);190    }191}192for (const base of pairs) {193  if (base.startsWith('research-')) {194    const slug = base.slice(9);195    if (slug !== 'index' && !noted.has(slug) && !there(`research/${slug}.md`)) drawn.push(`${base} shades no note`);196  }197  if (base.startsWith('demo-') && !demos.has(base.slice(5))) drawn.push(`${base} shades no demo`);198  if (base.startsWith('wiki-') && !there(`wiki/${base.slice(5)}.md`)) drawn.push(`${base} shades no wiki page`);199}200report('figures', `${pairs.length} pairs`, drawn);201202// LINKS203204const site = (target: string) => {205  const parts = target.split('/').filter((part) => part !== '');206  if (parts.length === 0) return true;207  const [head, next] = parts;208  if (head === 'research') {209    if (!next) return true;210    if (next === 'discoveries') return there('research/claims');211    return there(`research/notes/${next}.md`) || there(`research/${next}.md`);212  }213  if (head === 'demos') return !next || demos.has(next);214  if (head === 'papers') return !next || there(`research/papers/${next}.md`);215  if (head === 'blog') return !next || there(`site/blog/${next}.md`);216  if (head === 'wiki') return !next || there(`wiki/${next}.md`);217  if (head === 'tools' || head === 'math') return !next;218  if (head === 'method') return there('research/notes/method.md') || there('site/pages/method.md');219  return there(`site/pages/${head}.md`);220};221222const dead: string[] = [];223let aimedAt = 0;224for (const doc of prose) {225  const home = dirname(at(doc.name));226  for (const [n, line] of plain(doc))227    for (const hit of line.matchAll(/\[[^\]\n]*\]\(([^)\s]+)\)/g)) {228      const target = hit[1].split('#')[0];229      if (target === '' || /^(https?:|mailto:)/.test(target)) continue;230      const widget = target.match(/^demos\/([a-z0-9-]+)\/([a-z0-9-]+)$/);231      if (widget) {232        const file = `site/demos/${widget[1]}/widget.jsx`;233        if (!there(file)) dead.push(`${doc.name}:${n} ${target} has no widget.jsx`);234        else if (!new RegExp(`^export (?:function|const) ${widget[2]}\\b`, 'm').test(readFileSync(at(file), 'utf8'))) dead.push(`${doc.name}:${n} ${target} is not a view`);235        continue;236      }237      aimedAt += 1;238      if (target.startsWith('/')) {239        if (!site(target)) dead.push(`${doc.name}:${n} ${target}`);240      } else if (target.includes('/') || target.includes('.')) {241        if (!existsSync(resolve(home, target))) dead.push(`${doc.name}:${n} ${target}`);242      }243    }244}245report('links', `${aimedAt} links`, dead);246247// DEMOS248249const shown = new Set<string>();250for (const doc of [...notes, ...papers, ...pages, ...posts, ...concepts, ...(there('README.md') ? [read('README.md')] : [])])251  for (const hit of doc.lines.join('\n').matchAll(/demos\/([a-z0-9-]+)\//g)) shown.add(hit[1]);252const orphans = [...demos].filter((name) => !shown.has(name)).sort();253report('demos', `${demos.size} demos`, orphans.map((name) => `${name} is linked nowhere`));254255// NOTES256257const written: string[] = [];258for (const doc of notes) {259  const slug = stem(doc.name);260  for (const key of ['title', 'lead', 'figure', 'slug']) if (!doc.front.has(key)) written.push(`${doc.name} has no ${key}`);261  if (doc.front.get('slug') !== slug) written.push(`${doc.name} slug is ${doc.front.get('slug')}`);262  doc.lines.slice(doc.body).forEach((line, i) => {263    const where = `${doc.name}:${doc.body + i + 1}`;264    if (line.startsWith('# ')) written.push(`${where} carries an H1`);265    if (/\b20\d\d-\d\d-\d\d\b/.test(line)) written.push(`${where} carries a date`);266  });267}268for (const doc of [...notes, ...claims]) if (!/^[a-z0-9-]+$/.test(stem(doc.name))) written.push(`${doc.name} is not a slug`);269report('notes', `${notes.length} notes`, written);270271// WIKI272273const taught: string[] = [];274const slugs = new Set(concepts.map((doc) => stem(doc.name)));275for (const doc of concepts) {276  for (const key of ['title', 'lead']) if (!doc.front.has(key)) taught.push(`${doc.name} has no ${key}`);277  for (const need of (doc.front.get('prerequisites') ?? '').split(',').map((s) => s.trim()).filter(Boolean))278    if (!slugs.has(need)) taught.push(`${doc.name} needs ${need}, which has no page`);279  if (!doc.lines.slice(doc.body).some((line) => line.startsWith('## In the tree'))) taught.push(`${doc.name} never says where it appears in the tree`);280  doc.lines.slice(doc.body).forEach((line, i) => {281    const where = `${doc.name}:${doc.body + i + 1}`;282    if (line.startsWith('# ')) taught.push(`${where} carries an H1`);283    if (/\b20\d\d-\d\d-\d\d\b/.test(line)) taught.push(`${where} carries a date`);284    for (const hit of line.matchAll(/\]\(\/wiki\/([a-z0-9-]+)\/?\)/g)) if (!slugs.has(hit[1])) taught.push(`${where} links /wiki/${hit[1]}/, which has no page`);285  });286}287report('wiki', `${concepts.length} pages`, taught);288289// REFS290291const HEADER = '| ref | title | url |';292const RULE = '|---|---|---|';293const refs: string[] = [];294let cited = 0;295if (there('research/REFS.md')) {296  const doc = read('research/REFS.md');297  let section: string | null = null;298  let state: string | null = null;299  const seen = new Map<string, number>();300  doc.lines.forEach((line, i) => {301    const where = `research/REFS.md:${i + 1}`;302    if (line.startsWith('## ')) { section = line.slice(3).trim(); state = null; return; }303    if (line.startsWith('|')) {304      if (section === null || section === 'UNRESOLVED') { refs.push(`${where} table row outside a table section`); return; }305      if (state === null) { if (line !== HEADER) refs.push(`${where} table opens on the wrong header`); state = 'header'; return; }306      if (state === 'header') { if (line !== RULE) refs.push(`${where} header rule is wrong`); state = 'rows'; return; }307      const cells = line.split('|');308      if (cells.length !== 5 || cells[0] !== '' || cells[4] !== '') { refs.push(`${where} ${cells.length - 1} pipes`); return; }309      const [ref, title, url] = cells.slice(1, 4).map((cell) => cell.trim());310      cited += 1;311      if (!ref || !title) refs.push(`${where} empty ref or title`);312      if (!/^https?:\/\/\S+$/.test(url)) refs.push(`${where} url cell is not one URL`);313      if (seen.has(ref)) refs.push(`${where} ref ${ref} is already at line ${seen.get(ref)}`);314      seen.set(ref, i + 1);315      return;316    }317    if (line.startsWith('- ') && section !== null && section !== 'UNRESOLVED') refs.push(`${where} bullet inside ${section}`);318    else if (line !== '' && !line.startsWith('#') && state === 'rows') refs.push(`${where} stray line after the rows of ${section}`);319  });320} else refs.push('research/REFS.md is missing');321report('refs', `${cited} refs`, refs);322323// HOUSE324325const house: string[] = [];326let ruled = 0;327for (const doc of housed) {328  ruled += doc.lines.length;329  doc.lines.forEach((line, i) => {330    if (i > 0 && line === '' && doc.lines[i - 1] === '') house.push(`${doc.name}:${i + 1} two blank lines`);331  });332  for (const [n, line] of plain(doc)) {333    if (/[–—]/.test(line)) house.push(`${doc.name}:${n} em-dash or en-dash`);334    if (line.startsWith('#') && n < doc.lines.length && doc.lines[n] !== '') house.push(`${doc.name}:${n} heading without a blank line`);335  }336}337report('house', `${housed.length} files, ${ruled} lines`, house);338339// REGISTRIES340341const registries: string[] = [];342if (there('site/pages.json')) registries.push('site/pages.json still exists');343if (there('crates/mrlyfig/Cargo.toml') && readFileSync(at('crates/mrlyfig/Cargo.toml'), 'utf8').includes('[[example]]'))344  registries.push('mrlyfig Cargo.toml carries an [[example]]');345report('registries', 'none', registries);346347// WASM348349if (process.argv.includes('--wasm')) checks.push(...(await import('./check/wasm.ts')).default);350351let failed = 0;352for (const [label, got, want] of checks) {353  const ok = got === want;354  if (!ok) failed += 1;355  console.log(`${ok ? 'ok  ' : 'FAIL'} ${String(label).padEnd(26)} ${String(got)}${ok ? '' : ` (want ${String(want)})`}`);356}357console.log(failed ? `${failed} failed` : `${checks.length} checks green`);358console.log(`${(performance.now() - started).toFixed(0)} ms`);359process.exit(failed ? 1 : 0);