devadmin
99489ca17b
Deploy to Cloudflare Pages / deploy (push) Successful in 36s
- sync-catalog keeps a rolling 52-week history of index scores - the matrix shows the delta since the previous refresh plus a biggest-mover card
305 lines
10 KiB
JavaScript
305 lines
10 KiB
JavaScript
#!/usr/bin/env node
|
|
// Weekly evaluation refresh.
|
|
//
|
|
// node scripts/sync-catalog.mjs
|
|
//
|
|
// Pulls three things and regenerates the auto-generated parts of the site:
|
|
//
|
|
// 1. OpenCode Go catalogue -> models.dev/api.json, provider "opencode-go"
|
|
// (pricing, context window, modalities, release date, vendor description)
|
|
// 2. Benchmark scores -> Artificial Analysis model pages
|
|
// 3. Human-written layer -> site/assets/data.js (never touched by this script)
|
|
//
|
|
// Outputs
|
|
// site/assets/catalog.js catalogue metadata for every model currently in Go
|
|
// site/assets/evals.js benchmark scores per model
|
|
// site/assets/snapshot.json when the data was taken + what changed
|
|
// MODELS.md markdown rendering of the catalogue
|
|
//
|
|
// The human layer (site/assets/data.js) wins on anything it defines, so hand-written
|
|
// "targeted for" descriptions survive a refresh. New models show up with the vendor
|
|
// description and a NEEDS-WRITE-UP flag until someone writes them up.
|
|
|
|
import { writeFileSync } from "node:fs";
|
|
import { dirname, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
const UA =
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";
|
|
|
|
/* Artificial Analysis slug overrides where the slug is not the model id. */
|
|
const AA_OVERRIDES = {
|
|
"gemini-4-argon-contributor": "gemini-4-argon",
|
|
"beam-free": "reflection-beam",
|
|
"step-5-preview": "step-5",
|
|
"muse-spark-1.3-contributor": "muse-spark-1-3",
|
|
"muse-spark-1.2-contributor": "muse-spark-1-2",
|
|
"qwen3.8-flash": "qwen3-8-flash-next",
|
|
"qwen3.7-max": "qwen3-7-max",
|
|
"mimo-v2.5": "mimo-v2-5-0424",
|
|
"mimo-v2.5-pro": "mimo-v2-5-pro",
|
|
"mimo-v2.6-pro": "mimo-v2-6-pro",
|
|
"mimo-v2.6-flash": "mimo-v2-6-flash",
|
|
"minimax-m2.5": "minimax-m2-5",
|
|
"minimax-m2.7": "minimax-m2-7",
|
|
"hy3": "hy3",
|
|
"kimi-k2.6": "kimi-k2-6",
|
|
"glm-5.2": "glm-5-2",
|
|
"longcat-2.0": "longcat-2-0"
|
|
};
|
|
|
|
const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);
|
|
|
|
/* --- Artificial Analysis page parsing ------------------------------------ */
|
|
function flightPayload(html) {
|
|
const parts = [];
|
|
const re = /self\.__next_f\.push\(\[1,"/g;
|
|
let m;
|
|
while ((m = re.exec(html))) {
|
|
const start = re.lastIndex;
|
|
const end = html.indexOf('"])</script>', start);
|
|
if (end < 0) continue;
|
|
try {
|
|
parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
|
|
} catch {
|
|
/* ignore malformed chunk */
|
|
}
|
|
}
|
|
return parts.join("");
|
|
}
|
|
|
|
function extractModelObjects(html) {
|
|
const payload = flightPayload(html);
|
|
const out = {};
|
|
const re = /"slug":"/g;
|
|
let m;
|
|
while ((m = re.exec(payload))) {
|
|
const start = payload.lastIndexOf('{"id":"', m.index);
|
|
if (start < 0) continue;
|
|
let depth = 0, inStr = false, esc = false, k = start;
|
|
for (k = start; k < payload.length; k++) {
|
|
const c = payload[k];
|
|
if (esc) { esc = false; continue; }
|
|
if (c === "\\") { esc = true; continue; }
|
|
if (c === '"') { inStr = !inStr; continue; }
|
|
if (inStr) continue;
|
|
if (c === "{") depth++;
|
|
else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
|
|
}
|
|
let obj;
|
|
try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
|
|
if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function fetchAA(slug) {
|
|
const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
|
|
if (!res.ok) return null;
|
|
const models = extractModelObjects(await res.text());
|
|
return models[slug] || null;
|
|
}
|
|
|
|
function aaCandidates(id, canonical) {
|
|
const last = (canonical || id).split("/").pop();
|
|
const raw = [AA_OVERRIDES[id], id, last, id.replace(/-free$/, ""), id.replace(/-contributor$/, "")];
|
|
// Artificial Analysis slugs replace dots with dashes (deepseek-v4.1-flash -> deepseek-v4-1-flash)
|
|
const variants = [];
|
|
for (const c of raw) {
|
|
if (!c) continue;
|
|
variants.push(c);
|
|
const dashed = c.replace(/\./g, "-");
|
|
variants.push(dashed);
|
|
variants.push(dashed.replace(/-free$/, ""));
|
|
}
|
|
return [...new Set(variants)];
|
|
}
|
|
|
|
/* --- sources -------------------------------------------------------------- */
|
|
async function fetchCatalog() {
|
|
const res = await fetch("https://models.dev/api.json", { headers: { "user-agent": UA } });
|
|
if (!res.ok) throw new Error(`models.dev returned ${res.status}`);
|
|
const all = await res.json();
|
|
const go = all["opencode-go"];
|
|
if (!go) throw new Error('no "opencode-go" provider in models.dev/api.json');
|
|
return go.models;
|
|
}
|
|
|
|
function modalitiesOf(m) {
|
|
const inp = (m.modalities?.input || ["text"]).join(", ");
|
|
const out = (m.modalities?.output || ["text"]).join(", ");
|
|
return `${inp} in → ${out} out`;
|
|
}
|
|
|
|
async function fetchLiveGoIds() {
|
|
try {
|
|
const res = await fetch("https://opencode.ai/inference/go/v1/models", { headers: { "user-agent": UA } });
|
|
if (!res.ok) return null;
|
|
const j = await res.json();
|
|
return (j.data || []).map((m) => m.id);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/* --- main ------------------------------------------------------------------ */
|
|
const catalogModels = await fetchCatalog();
|
|
const liveIds = await fetchLiveGoIds();
|
|
const ids = Object.keys(catalogModels);
|
|
console.log(`catalog: ${ids.length} models in the OpenCode Go provider`);
|
|
|
|
const catalog = [];
|
|
const scores = [];
|
|
|
|
for (const id of ids) {
|
|
const m = catalogModels[id];
|
|
const cost = m.cost || {};
|
|
const free = !cost.input && !cost.output;
|
|
const canonical = m.canonical_model_id || null;
|
|
|
|
let aa = null;
|
|
for (const slug of aaCandidates(id, canonical)) {
|
|
aa = await fetchAA(slug);
|
|
if (aa) break;
|
|
}
|
|
|
|
const row = {
|
|
id,
|
|
n: m.name || id,
|
|
lab: canonical ? canonical.split("/")[0] : "",
|
|
released: m.release_date || "",
|
|
price: { in: cost.input ?? 0, out: cost.output ?? 0 },
|
|
free,
|
|
ctx: m.limit?.context ?? null,
|
|
modalities: modalitiesOf(m),
|
|
description: m.description || "",
|
|
canonical,
|
|
openWeights: !!m.open_weights
|
|
};
|
|
catalog.push(row);
|
|
|
|
scores.push({
|
|
id,
|
|
n: row.n,
|
|
lab: row.lab,
|
|
free,
|
|
pi: row.price.in,
|
|
po: row.price.out,
|
|
ii: aa ? Math.round(aa.intelligenceIndex * 10) / 10 : null,
|
|
fc: pct(aa?.frontierCode),
|
|
scicode: pct(aa?.scicode),
|
|
tb21: pct(aa?.terminalBench21),
|
|
tb40: pct(aa?.terminalBench40),
|
|
tbh: pct(aa?.terminalbenchHard),
|
|
tau2: pct(aa?.tau2),
|
|
taub: pct(aa?.tauBanking),
|
|
apex: pct(aa?.apexAgents),
|
|
analyst: pct(aa?.analystAgent),
|
|
auto: pct(aa?.automationBenchPartialScore),
|
|
gpqa: pct(aa?.gpqa),
|
|
critpt: pct(aa?.critpt),
|
|
hle: pct(aa?.hle),
|
|
lcr: pct(aa?.lcr),
|
|
ifbench: pct(aa?.ifbench),
|
|
mmmu: pct(aa?.mmmuPro),
|
|
gdpv: pct(aa?.gdpvalNormalized),
|
|
gdppdf: pct(aa?.gdpPdfAllPass),
|
|
ctx: row.ctx ?? 1048576
|
|
});
|
|
console.log(` ${aa ? "+" : "-"} ${id.padEnd(32)} ${aa ? "index " + aa.intelligenceIndex.toFixed(1) : "no public evals"}`);
|
|
}
|
|
|
|
/* --- diff against the previous snapshot ---------------------------------- */
|
|
let previous = { models: [] };
|
|
try {
|
|
previous = JSON.parse(
|
|
await (await import("node:fs/promises")).readFile(resolve(ROOT, "site/assets/snapshot.json"), "utf8")
|
|
);
|
|
} catch {
|
|
/* first run */
|
|
}
|
|
const prevIds = previous.models || [];
|
|
const newModels = ids.filter((i) => !prevIds.includes(i));
|
|
const removedModels = prevIds.filter((i) => !ids.includes(i));
|
|
const prevScores = Object.fromEntries((previous.scores || []).map((s) => [s.id, s.ii]));
|
|
const deltas = scores
|
|
.filter((s) => s.ii !== null && prevScores[s.id] !== undefined && prevScores[s.id] !== null)
|
|
.map((s) => ({ id: s.id, from: prevScores[s.id], to: s.ii, delta: Math.round((s.ii - prevScores[s.id]) * 10) / 10 }))
|
|
.filter((d) => d.delta !== 0)
|
|
.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
|
|
|
|
const snapshot = {
|
|
generated: new Date().toISOString(),
|
|
models: ids,
|
|
scores: scores.map((s) => ({ id: s.id, ii: s.ii })),
|
|
newModels,
|
|
removedModels,
|
|
deltas,
|
|
liveCatalogSize: liveIds ? liveIds.length : null,
|
|
notInLiveCatalog: liveIds ? ids.filter((i) => !liveIds.includes(i)) : []
|
|
};
|
|
|
|
/* --- write files ---------------------------------------------------------- */
|
|
writeFileSync(
|
|
resolve(ROOT, "site/assets/catalog.js"),
|
|
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
|
// Source: models.dev/api.json (provider "opencode-go") + Artificial Analysis.
|
|
export const CATALOG = ${JSON.stringify(catalog, null, 1)};
|
|
|
|
export const SNAPSHOT = ${JSON.stringify(
|
|
{
|
|
generated: snapshot.generated,
|
|
models: snapshot.models.length,
|
|
newModels: snapshot.newModels,
|
|
removedModels: snapshot.removedModels,
|
|
deltas: snapshot.deltas,
|
|
notInLiveCatalog: snapshot.notInLiveCatalog
|
|
},
|
|
null,
|
|
1
|
|
)};
|
|
`
|
|
);
|
|
|
|
writeFileSync(
|
|
resolve(ROOT, "site/assets/evals.js"),
|
|
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
|
// Benchmark scores from Artificial Analysis (single consistent harness), scaled to 0-100.
|
|
export const RAW = ${JSON.stringify(scores, null, 1)};
|
|
`
|
|
);
|
|
|
|
writeFileSync(resolve(ROOT, "site/assets/snapshot.json"), JSON.stringify(snapshot, null, 2) + "\n");
|
|
|
|
/* --- rolling history (one entry per refresh, last 52 weeks) ---------------- */
|
|
let history = [];
|
|
try {
|
|
history = JSON.parse(await (await import("node:fs/promises")).readFile(resolve(ROOT, "site/assets/history.json"), "utf8"));
|
|
} catch {
|
|
/* first run */
|
|
}
|
|
const week = snapshot.generated.slice(0, 10);
|
|
if (!history.length || history[0].date !== week) {
|
|
history.unshift({
|
|
date: week,
|
|
scores: Object.fromEntries(scores.filter((s) => s.ii !== null).map((s) => [s.id, s.ii]))
|
|
});
|
|
history = history.slice(0, 52);
|
|
}
|
|
writeFileSync(resolve(ROOT, "site/assets/history.json"), JSON.stringify(history, null, 1) + "\n");
|
|
writeFileSync(
|
|
resolve(ROOT, "site/assets/history.js"),
|
|
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.\n// Rolling week-over-week snapshot of the Intelligence Index.\nexport const HISTORY = ${JSON.stringify(history, null, 1)};\n`
|
|
);
|
|
|
|
console.log("\nnew models:", newModels.length ? newModels.join(", ") : "none");
|
|
console.log("removed models:", removedModels.length ? removedModels.join(", ") : "none");
|
|
if (deltas.length) {
|
|
console.log("score changes:");
|
|
for (const d of deltas) console.log(` ${d.id}: ${d.from} → ${d.to} (${d.delta > 0 ? "+" : ""}${d.delta})`);
|
|
} else {
|
|
console.log("score changes: none");
|
|
}
|
|
console.log("\nwrote site/assets/catalog.js, evals.js, snapshot.json, history.json");
|