OpenCodeGoEvals/scripts/sync-catalog.mjs

#!/usr/bin/env node
// Weekly evaluation refresh.
//
//   node scripts/sync-catalog.mjs
//
// Pulls three things and regenerates the auto-generated parts of the site:
//
//   1. OpenCode Go catalogue  -> models.dev/api.json, provider "opencode-go"
//      (pricing, context window, modalities, release date, vendor description)
//   2. Benchmark scores       -> Artificial Analysis model pages
//   3. Human-written layer    -> site/assets/data.js (never touched by this script)
//
// Outputs
//   site/assets/catalog.js   catalogue metadata for every model currently in Go
//   site/assets/evals.js     benchmark scores per model
//   site/assets/snapshot.json  when the data was taken + what changed
//   MODELS.md                markdown rendering of the catalogue
//
// The human layer (site/assets/data.js) wins on anything it defines, so hand-written
// "targeted for" descriptions survive a refresh. New models show up with the vendor
// description and a NEEDS-WRITE-UP flag until someone writes them up.

import { writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const UA =
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";

/* Artificial Analysis slug overrides where the slug is not the model id. */
const AA_OVERRIDES = {
  "gemini-4-argon-contributor": "gemini-4-argon",
  "beam-free": "reflection-beam",
  "step-5-preview": "step-5",
  "muse-spark-1.3-contributor": "muse-spark-1-3",
  "muse-spark-1.2-contributor": "muse-spark-1-2",
  "qwen3.8-flash": "qwen3-8-flash-next",
  "qwen3.7-max": "qwen3-7-max",
  "mimo-v2.5": "mimo-v2-5-0424",
  "mimo-v2.5-pro": "mimo-v2-5-pro",
  "mimo-v2.6-pro": "mimo-v2-6-pro",
  "mimo-v2.6-flash": "mimo-v2-6-flash",
  "minimax-m2.5": "minimax-m2-5",
  "minimax-m2.7": "minimax-m2-7",
  "hy3": "hy3",
  "kimi-k2.6": "kimi-k2-6",
  "glm-5.2": "glm-5-2",
  "longcat-2.0": "longcat-2-0"
};

const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);

/* --- Artificial Analysis page parsing ------------------------------------ */
function flightPayload(html) {
  const parts = [];
  const re = /self\.__next_f\.push\(\[1,"/g;
  let m;
  while ((m = re.exec(html))) {
    const start = re.lastIndex;
    const end = html.indexOf('"])</script>', start);
    if (end < 0) continue;
    try {
      parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
    } catch {
      /* ignore malformed chunk */
    }
  }
  return parts.join("");
}

function extractModelObjects(html) {
  const payload = flightPayload(html);
  const out = {};
  const re = /"slug":"/g;
  let m;
  while ((m = re.exec(payload))) {
    const start = payload.lastIndexOf('{"id":"', m.index);
    if (start < 0) continue;
    let depth = 0, inStr = false, esc = false, k = start;
    for (k = start; k < payload.length; k++) {
      const c = payload[k];
      if (esc) { esc = false; continue; }
      if (c === "\\") { esc = true; continue; }
      if (c === '"') { inStr = !inStr; continue; }
      if (inStr) continue;
      if (c === "{") depth++;
      else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
    }
    let obj;
    try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
    if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
  }
  return out;
}

async function fetchAA(slug) {
  const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
  if (!res.ok) return null;
  const models = extractModelObjects(await res.text());
  return models[slug] || null;
}

function aaCandidates(id, canonical) {
  const last = (canonical || id).split("/").pop();
  const raw = [AA_OVERRIDES[id], id, last, id.replace(/-free$/, ""), id.replace(/-contributor$/, "")];
  // Artificial Analysis slugs replace dots with dashes (deepseek-v4.1-flash -> deepseek-v4-1-flash)
  const variants = [];
  for (const c of raw) {
    if (!c) continue;
    variants.push(c);
    const dashed = c.replace(/\./g, "-");
    variants.push(dashed);
    variants.push(dashed.replace(/-free$/, ""));
  }
  return [...new Set(variants)];
}

/* --- sources -------------------------------------------------------------- */
async function fetchCatalog() {
  const res = await fetch("https://models.dev/api.json", { headers: { "user-agent": UA } });
  if (!res.ok) throw new Error(`models.dev returned ${res.status}`);
  const all = await res.json();
  const go = all["opencode-go"];
  if (!go) throw new Error('no "opencode-go" provider in models.dev/api.json');
  return go.models;
}

function modalitiesOf(m) {
  const inp = (m.modalities?.input || ["text"]).join(", ");
  const out = (m.modalities?.output || ["text"]).join(", ");
  return `${inp} in → ${out} out`;
}

async function fetchLiveGoIds() {
  try {
    const res = await fetch("https://opencode.ai/inference/go/v1/models", { headers: { "user-agent": UA } });
    if (!res.ok) return null;
    const j = await res.json();
    return (j.data || []).map((m) => m.id);
  } catch {
    return null;
  }
}

/* --- main ------------------------------------------------------------------ */
const catalogModels = await fetchCatalog();
const liveIds = await fetchLiveGoIds();
const ids = Object.keys(catalogModels);
console.log(`catalog: ${ids.length} models in the OpenCode Go provider`);

const catalog = [];
const scores = [];

for (const id of ids) {
  const m = catalogModels[id];
  const cost = m.cost || {};
  const free = !cost.input && !cost.output;
  const canonical = m.canonical_model_id || null;

  let aa = null;
  for (const slug of aaCandidates(id, canonical)) {
    aa = await fetchAA(slug);
    if (aa) break;
  }

  const row = {
    id,
    n: m.name || id,
    lab: canonical ? canonical.split("/")[0] : "",
    released: m.release_date || "",
    price: { in: cost.input ?? 0, out: cost.output ?? 0 },
    free,
    ctx: m.limit?.context ?? null,
    modalities: modalitiesOf(m),
    description: m.description || "",
    canonical,
    openWeights: !!m.open_weights
  };
  catalog.push(row);

  scores.push({
    id,
    n: row.n,
    lab: row.lab,
    free,
    pi: row.price.in,
    po: row.price.out,
    ii: aa ? Math.round(aa.intelligenceIndex * 10) / 10 : null,
    fc: pct(aa?.frontierCode),
    scicode: pct(aa?.scicode),
    tb21: pct(aa?.terminalBench21),
    tb40: pct(aa?.terminalBench40),
    tbh: pct(aa?.terminalbenchHard),
    tau2: pct(aa?.tau2),
    taub: pct(aa?.tauBanking),
    apex: pct(aa?.apexAgents),
    analyst: pct(aa?.analystAgent),
    auto: pct(aa?.automationBenchPartialScore),
    gpqa: pct(aa?.gpqa),
    critpt: pct(aa?.critpt),
    hle: pct(aa?.hle),
    lcr: pct(aa?.lcr),
    ifbench: pct(aa?.ifbench),
    mmmu: pct(aa?.mmmuPro),
    gdpv: pct(aa?.gdpvalNormalized),
    gdppdf: pct(aa?.gdpPdfAllPass),
    ctx: row.ctx ?? 1048576
  });
  console.log(`  ${aa ? "+" : "-"} ${id.padEnd(32)} ${aa ? "index " + aa.intelligenceIndex.toFixed(1) : "no public evals"}`);
}

/* --- diff against the previous snapshot ---------------------------------- */
let previous = { models: [] };
try {
  previous = JSON.parse(
    await (await import("node:fs/promises")).readFile(resolve(ROOT, "site/assets/snapshot.json"), "utf8")
  );
} catch {
  /* first run */
}
const prevIds = previous.models || [];
const newModels = ids.filter((i) => !prevIds.includes(i));
const removedModels = prevIds.filter((i) => !ids.includes(i));
const prevScores = Object.fromEntries((previous.scores || []).map((s) => [s.id, s.ii]));
const deltas = scores
  .filter((s) => s.ii !== null && prevScores[s.id] !== undefined && prevScores[s.id] !== null)
  .map((s) => ({ id: s.id, from: prevScores[s.id], to: s.ii, delta: Math.round((s.ii - prevScores[s.id]) * 10) / 10 }))
  .filter((d) => d.delta !== 0)
  .sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));

const snapshot = {
  generated: new Date().toISOString(),
  models: ids,
  scores: scores.map((s) => ({ id: s.id, ii: s.ii })),
  newModels,
  removedModels,
  deltas,
  liveCatalogSize: liveIds ? liveIds.length : null,
  notInLiveCatalog: liveIds ? ids.filter((i) => !liveIds.includes(i)) : []
};

/* --- write files ---------------------------------------------------------- */
writeFileSync(
  resolve(ROOT, "site/assets/catalog.js"),
  `// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
// Source: models.dev/api.json (provider "opencode-go") + Artificial Analysis.
export const CATALOG = ${JSON.stringify(catalog, null, 1)};

export const SNAPSHOT = ${JSON.stringify(
    {
      generated: snapshot.generated,
      models: snapshot.models.length,
      newModels: snapshot.newModels,
      removedModels: snapshot.removedModels,
      deltas: snapshot.deltas,
      notInLiveCatalog: snapshot.notInLiveCatalog
    },
    null,
    1
  )};
`
);

writeFileSync(
  resolve(ROOT, "site/assets/evals.js"),
  `// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
// Benchmark scores from Artificial Analysis (single consistent harness), scaled to 0-100.
export const RAW = ${JSON.stringify(scores, null, 1)};
`
);

writeFileSync(resolve(ROOT, "site/assets/snapshot.json"), JSON.stringify(snapshot, null, 2) + "\n");

console.log("\nnew models:", newModels.length ? newModels.join(", ") : "none");
console.log("removed models:", removedModels.length ? removedModels.join(", ") : "none");
if (deltas.length) {
  console.log("score changes:");
  for (const d of deltas) console.log(`  ${d.id}: ${d.from} → ${d.to} (${d.delta > 0 ? "+" : ""}${d.delta})`);
} else {
  console.log("score changes: none");
}
console.log("\nwrote site/assets/catalog.js, evals.js, snapshot.json");