Automate the evaluation on a weekly schedule
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
This commit is contained in:
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env node
|
||||
// Weekly evaluation refresh.
|
||||
//
|
||||
// node scripts/sync-catalog.mjs
|
||||
//
|
||||
// Pulls three things and regenerates the auto-generated parts of the site:
|
||||
//
|
||||
// 1. OpenCode Go catalogue -> models.dev/api.json, provider "opencode-go"
|
||||
// (pricing, context window, modalities, release date, vendor description)
|
||||
// 2. Benchmark scores -> Artificial Analysis model pages
|
||||
// 3. Human-written layer -> site/assets/data.js (never touched by this script)
|
||||
//
|
||||
// Outputs
|
||||
// site/assets/catalog.js catalogue metadata for every model currently in Go
|
||||
// site/assets/evals.js benchmark scores per model
|
||||
// site/assets/snapshot.json when the data was taken + what changed
|
||||
// MODELS.md markdown rendering of the catalogue
|
||||
//
|
||||
// The human layer (site/assets/data.js) wins on anything it defines, so hand-written
|
||||
// "targeted for" descriptions survive a refresh. New models show up with the vendor
|
||||
// description and a NEEDS-WRITE-UP flag until someone writes them up.
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";
|
||||
|
||||
/* Artificial Analysis slug overrides where the slug is not the model id. */
|
||||
const AA_OVERRIDES = {
|
||||
"gemini-4-argon-contributor": "gemini-4-argon",
|
||||
"beam-free": "reflection-beam",
|
||||
"step-5-preview": "step-5",
|
||||
"muse-spark-1.3-contributor": "muse-spark-1-3",
|
||||
"muse-spark-1.2-contributor": "muse-spark-1-2",
|
||||
"qwen3.8-flash": "qwen3-8-flash-next",
|
||||
"qwen3.7-max": "qwen3-7-max",
|
||||
"mimo-v2.5": "mimo-v2-5-0424",
|
||||
"mimo-v2.5-pro": "mimo-v2-5-pro",
|
||||
"mimo-v2.6-pro": "mimo-v2-6-pro",
|
||||
"mimo-v2.6-flash": "mimo-v2-6-flash",
|
||||
"minimax-m2.5": "minimax-m2-5",
|
||||
"minimax-m2.7": "minimax-m2-7",
|
||||
"hy3": "hy3",
|
||||
"kimi-k2.6": "kimi-k2-6",
|
||||
"glm-5.2": "glm-5-2",
|
||||
"longcat-2.0": "longcat-2-0"
|
||||
};
|
||||
|
||||
const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);
|
||||
|
||||
/* --- Artificial Analysis page parsing ------------------------------------ */
|
||||
function flightPayload(html) {
|
||||
const parts = [];
|
||||
const re = /self\.__next_f\.push\(\[1,"/g;
|
||||
let m;
|
||||
while ((m = re.exec(html))) {
|
||||
const start = re.lastIndex;
|
||||
const end = html.indexOf('"])</script>', start);
|
||||
if (end < 0) continue;
|
||||
try {
|
||||
parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
|
||||
} catch {
|
||||
/* ignore malformed chunk */
|
||||
}
|
||||
}
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function extractModelObjects(html) {
|
||||
const payload = flightPayload(html);
|
||||
const out = {};
|
||||
const re = /"slug":"/g;
|
||||
let m;
|
||||
while ((m = re.exec(payload))) {
|
||||
const start = payload.lastIndexOf('{"id":"', m.index);
|
||||
if (start < 0) continue;
|
||||
let depth = 0, inStr = false, esc = false, k = start;
|
||||
for (k = start; k < payload.length; k++) {
|
||||
const c = payload[k];
|
||||
if (esc) { esc = false; continue; }
|
||||
if (c === "\\") { esc = true; continue; }
|
||||
if (c === '"') { inStr = !inStr; continue; }
|
||||
if (inStr) continue;
|
||||
if (c === "{") depth++;
|
||||
else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
|
||||
}
|
||||
let obj;
|
||||
try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
|
||||
if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function fetchAA(slug) {
|
||||
const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
|
||||
if (!res.ok) return null;
|
||||
const models = extractModelObjects(await res.text());
|
||||
return models[slug] || null;
|
||||
}
|
||||
|
||||
function aaCandidates(id, canonical) {
|
||||
const last = (canonical || id).split("/").pop();
|
||||
const raw = [AA_OVERRIDES[id], id, last, id.replace(/-free$/, ""), id.replace(/-contributor$/, "")];
|
||||
// Artificial Analysis slugs replace dots with dashes (deepseek-v4.1-flash -> deepseek-v4-1-flash)
|
||||
const variants = [];
|
||||
for (const c of raw) {
|
||||
if (!c) continue;
|
||||
variants.push(c);
|
||||
const dashed = c.replace(/\./g, "-");
|
||||
variants.push(dashed);
|
||||
variants.push(dashed.replace(/-free$/, ""));
|
||||
}
|
||||
return [...new Set(variants)];
|
||||
}
|
||||
|
||||
/* --- sources -------------------------------------------------------------- */
|
||||
async function fetchCatalog() {
|
||||
const res = await fetch("https://models.dev/api.json", { headers: { "user-agent": UA } });
|
||||
if (!res.ok) throw new Error(`models.dev returned ${res.status}`);
|
||||
const all = await res.json();
|
||||
const go = all["opencode-go"];
|
||||
if (!go) throw new Error('no "opencode-go" provider in models.dev/api.json');
|
||||
return go.models;
|
||||
}
|
||||
|
||||
function modalitiesOf(m) {
|
||||
const inp = (m.modalities?.input || ["text"]).join(", ");
|
||||
const out = (m.modalities?.output || ["text"]).join(", ");
|
||||
return `${inp} in → ${out} out`;
|
||||
}
|
||||
|
||||
async function fetchLiveGoIds() {
|
||||
try {
|
||||
const res = await fetch("https://opencode.ai/inference/go/v1/models", { headers: { "user-agent": UA } });
|
||||
if (!res.ok) return null;
|
||||
const j = await res.json();
|
||||
return (j.data || []).map((m) => m.id);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/* --- main ------------------------------------------------------------------ */
|
||||
const catalogModels = await fetchCatalog();
|
||||
const liveIds = await fetchLiveGoIds();
|
||||
const ids = Object.keys(catalogModels);
|
||||
console.log(`catalog: ${ids.length} models in the OpenCode Go provider`);
|
||||
|
||||
const catalog = [];
|
||||
const scores = [];
|
||||
|
||||
for (const id of ids) {
|
||||
const m = catalogModels[id];
|
||||
const cost = m.cost || {};
|
||||
const free = !cost.input && !cost.output;
|
||||
const canonical = m.canonical_model_id || null;
|
||||
|
||||
let aa = null;
|
||||
for (const slug of aaCandidates(id, canonical)) {
|
||||
aa = await fetchAA(slug);
|
||||
if (aa) break;
|
||||
}
|
||||
|
||||
const row = {
|
||||
id,
|
||||
n: m.name || id,
|
||||
lab: canonical ? canonical.split("/")[0] : "",
|
||||
released: m.release_date || "",
|
||||
price: { in: cost.input ?? 0, out: cost.output ?? 0 },
|
||||
free,
|
||||
ctx: m.limit?.context ?? null,
|
||||
modalities: modalitiesOf(m),
|
||||
description: m.description || "",
|
||||
canonical,
|
||||
openWeights: !!m.open_weights
|
||||
};
|
||||
catalog.push(row);
|
||||
|
||||
scores.push({
|
||||
id,
|
||||
n: row.n,
|
||||
lab: row.lab,
|
||||
free,
|
||||
pi: row.price.in,
|
||||
po: row.price.out,
|
||||
ii: aa ? Math.round(aa.intelligenceIndex * 10) / 10 : null,
|
||||
fc: pct(aa?.frontierCode),
|
||||
scicode: pct(aa?.scicode),
|
||||
tb21: pct(aa?.terminalBench21),
|
||||
tb40: pct(aa?.terminalBench40),
|
||||
tbh: pct(aa?.terminalbenchHard),
|
||||
tau2: pct(aa?.tau2),
|
||||
taub: pct(aa?.tauBanking),
|
||||
apex: pct(aa?.apexAgents),
|
||||
analyst: pct(aa?.analystAgent),
|
||||
auto: pct(aa?.automationBenchPartialScore),
|
||||
gpqa: pct(aa?.gpqa),
|
||||
critpt: pct(aa?.critpt),
|
||||
hle: pct(aa?.hle),
|
||||
lcr: pct(aa?.lcr),
|
||||
ifbench: pct(aa?.ifbench),
|
||||
mmmu: pct(aa?.mmmuPro),
|
||||
gdpv: pct(aa?.gdpvalNormalized),
|
||||
gdppdf: pct(aa?.gdpPdfAllPass),
|
||||
ctx: row.ctx ?? 1048576
|
||||
});
|
||||
console.log(` ${aa ? "+" : "-"} ${id.padEnd(32)} ${aa ? "index " + aa.intelligenceIndex.toFixed(1) : "no public evals"}`);
|
||||
}
|
||||
|
||||
/* --- diff against the previous snapshot ---------------------------------- */
|
||||
let previous = { models: [] };
|
||||
try {
|
||||
previous = JSON.parse(
|
||||
await (await import("node:fs/promises")).readFile(resolve(ROOT, "site/assets/snapshot.json"), "utf8")
|
||||
);
|
||||
} catch {
|
||||
/* first run */
|
||||
}
|
||||
const prevIds = previous.models || [];
|
||||
const newModels = ids.filter((i) => !prevIds.includes(i));
|
||||
const removedModels = prevIds.filter((i) => !ids.includes(i));
|
||||
const prevScores = Object.fromEntries((previous.scores || []).map((s) => [s.id, s.ii]));
|
||||
const deltas = scores
|
||||
.filter((s) => s.ii !== null && prevScores[s.id] !== undefined && prevScores[s.id] !== null)
|
||||
.map((s) => ({ id: s.id, from: prevScores[s.id], to: s.ii, delta: Math.round((s.ii - prevScores[s.id]) * 10) / 10 }))
|
||||
.filter((d) => d.delta !== 0)
|
||||
.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
|
||||
|
||||
const snapshot = {
|
||||
generated: new Date().toISOString(),
|
||||
models: ids,
|
||||
scores: scores.map((s) => ({ id: s.id, ii: s.ii })),
|
||||
newModels,
|
||||
removedModels,
|
||||
deltas,
|
||||
liveCatalogSize: liveIds ? liveIds.length : null,
|
||||
notInLiveCatalog: liveIds ? ids.filter((i) => !liveIds.includes(i)) : []
|
||||
};
|
||||
|
||||
/* --- write files ---------------------------------------------------------- */
|
||||
writeFileSync(
|
||||
resolve(ROOT, "site/assets/catalog.js"),
|
||||
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
||||
// Source: models.dev/api.json (provider "opencode-go") + Artificial Analysis.
|
||||
export const CATALOG = ${JSON.stringify(catalog, null, 1)};
|
||||
|
||||
export const SNAPSHOT = ${JSON.stringify(
|
||||
{
|
||||
generated: snapshot.generated,
|
||||
models: snapshot.models.length,
|
||||
newModels: snapshot.newModels,
|
||||
removedModels: snapshot.removedModels,
|
||||
deltas: snapshot.deltas,
|
||||
notInLiveCatalog: snapshot.notInLiveCatalog
|
||||
},
|
||||
null,
|
||||
1
|
||||
)};
|
||||
`
|
||||
);
|
||||
|
||||
writeFileSync(
|
||||
resolve(ROOT, "site/assets/evals.js"),
|
||||
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
||||
// Benchmark scores from Artificial Analysis (single consistent harness), scaled to 0-100.
|
||||
export const RAW = ${JSON.stringify(scores, null, 1)};
|
||||
`
|
||||
);
|
||||
|
||||
writeFileSync(resolve(ROOT, "site/assets/snapshot.json"), JSON.stringify(snapshot, null, 2) + "\n");
|
||||
|
||||
console.log("\nnew models:", newModels.length ? newModels.join(", ") : "none");
|
||||
console.log("removed models:", removedModels.length ? removedModels.join(", ") : "none");
|
||||
if (deltas.length) {
|
||||
console.log("score changes:");
|
||||
for (const d of deltas) console.log(` ${d.id}: ${d.from} → ${d.to} (${d.delta > 0 ? "+" : ""}${d.delta})`);
|
||||
} else {
|
||||
console.log("score changes: none");
|
||||
}
|
||||
console.log("\nwrote site/assets/catalog.js, evals.js, snapshot.json");
|
||||
Reference in New Issue
Block a user