Automate the evaluation on a weekly schedule
Deploy to Cloudflare Pages / deploy (push) Successful in 30s

- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark
  scores from Artificial Analysis, and records what changed in snapshot.json
- scripts/build-models-md.mjs: regenerates MODELS.md from the data layers
- scripts/notify-new-models.mjs: files an issue when a new model needs a write-up
- weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy
- site now renders every model in the catalogue (33) with a data-snapshot stamp;
  auto-added models are flagged 'no write-up' until described
- replaced scripts/refresh-evals.mjs with the wider sync script
This commit is contained in:
2026-10-21 17:06:51 +00:00
parent e76281876b
commit 4e5a13f7b6
13 changed files with 2934 additions and 610 deletions
+65 -38
View File
@@ -1,7 +1,34 @@
import { CATALOG, SNAPSHOT } from "./catalog.js";
import { RAW } from "./evals.js";
import { GO_MODELS, FREE_NOTES, CATEGORIES } from "./data.js";
import { GO_MODELS, FREE_NOTES } from "./data.js";
/* ---------- shared helpers ---------- */
/* ---------- merge the three layers -----------------------------------------
catalog.js auto-generated — every model currently in OpenCode Go
evals.js auto-generated — benchmark scores per model
data.js hand-written — what each model is targeted for
--------------------------------------------------------------------------- */
const DESC = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));
const SCORES = Object.fromEntries(RAW.map((m) => [m.id, m]));
const MODELS = CATALOG.map((c) => {
const d = DESC[c.id] || {};
const s = SCORES[c.id] || {};
const m = {
...c,
best: d.best || [],
blurb: d.blurb || c.description || "No description published yet.",
target: d.target || "",
caveats: d.caveats || [],
written: !!d.blurb,
ii: s.ii ?? null,
tasks: {},
src: {}
};
m.blend = m.free ? 0 : 0.7 * m.price.in + 0.3 * m.price.out;
return m;
});
/* ---------- shared helpers ------------------------------------------------ */
const el = (id) => document.getElementById(id);
const tip = el("tip");
function showTip(html, e) {
@@ -40,32 +67,31 @@ const TASKS = [
{ key: "pro", label: "Professional work", picks: [["gdpv", "GDPval-AA"], ["gdppdf", "GDP-PDF"]] }
];
RAW.forEach((m) => {
m.tasks = {}; m.src = {};
MODELS.forEach((m) => {
TASKS.forEach((t) => {
for (const [f, label] of t.picks) {
const v = m[f];
const v = SCORES[m.id]?.[f];
if (v !== null && v !== undefined) { m.tasks[t.key] = v; m.src[t.key] = label; break; }
}
});
m.blend = m.free ? 0 : 0.7 * m.pi + 0.3 * m.po;
});
const byId = Object.fromEntries(RAW.map((m) => [m.id, m]));
/* ---------- header cards ---------- */
const scored = RAW.filter((m) => m.ii !== null);
const scored = MODELS.filter((m) => m.ii !== null);
const topOverall = scored.slice().sort((a, b) => b.ii - a.ii)[0];
const bestValue = scored.filter((m) => !m.free).slice().sort((a, b) => b.ii / b.blend - a.ii / a.blend)[0];
const snapshotDate = (SNAPSHOT.generated || "").slice(0, 10);
/* ---------- header cards -------------------------------------------------- */
el("cards").innerHTML = [
["Models in Go", RAW.length, "in the Go catalog"],
["Models in Go", MODELS.length, "in the OpenCode Go catalogue"],
["With public evals", scored.length, "Artificial Analysis harness"],
["Top overall", topOverall.n, "index " + topOverall.ii.toFixed(1)],
["Best value", bestValue.n, "index " + bestValue.ii.toFixed(1) + " at $" + bestValue.blend.toFixed(2) + "/1M"],
["Free tier", RAW.filter((m) => m.free).length, "limited-time free models"]
["Free tier", MODELS.filter((m) => m.free).length, "limited-time free models"],
["Data snapshot", snapshotDate, "refreshed weekly by CI"]
].map(([k, v, s]) => `<div class="card"><div class="k">${k}</div><div class="v">${v}</div><div class="sub" style="font-size:11px">${s}</div></div>`).join("");
/* ---------- matrix ---------- */
/* ---------- matrix --------------------------------------------------------- */
const cols = [
{ key: "name", label: "Model", get: (m) => m.n, txt: true },
{ key: "ii", label: "Index", get: (m) => m.ii },
@@ -78,10 +104,10 @@ function renderMatrix() {
const best = {};
cols.forEach((c) => {
if (c.key === "name" || c.key === "blend") return;
const vals = RAW.map((m) => c.get(m)).filter((v) => v !== null && v !== undefined);
const vals = MODELS.map((m) => c.get(m)).filter((v) => v !== null && v !== undefined);
best[c.key] = vals.length ? Math.max(...vals) : null;
});
const rows = RAW.slice().sort((a, b) => {
const rows = MODELS.slice().sort((a, b) => {
const c = cols.find((x) => x.key === sortKey);
const va = c.get(a), vb = c.get(b);
if (sortKey === "name") return String(va).localeCompare(String(vb)) * sortDir;
@@ -95,7 +121,8 @@ function renderMatrix() {
});
h += "</tr></thead><tbody>";
rows.forEach((m) => {
h += `<tr><td class="name">${m.n}${m.free ? '<span class="tag free">free</span>' : ""}<div class="price">${m.lab} · $${m.pi}/${m.po} · ${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M ctx${m.ii === null ? " · no public evals" : ""}</div></td>`;
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M ctx` : "ctx n/a";
h += `<tr><td class="name">${m.n}${m.free ? '<span class="tag free">free</span>' : ""}${!m.written ? '<span class="tag warn">no write-up</span>' : ""}<div class="price">${m.lab || "—"} · $${m.price.in}/${m.price.out} · ${ctx}${m.ii === null ? " · no public evals" : ""}</div></td>`;
cols.forEach((c) => {
if (c.key === "name") return;
const v = c.get(m);
@@ -127,7 +154,7 @@ function renderMatrix() {
}
renderMatrix();
/* ---------- svg helpers ---------- */
/* ---------- svg helpers ---------------------------------------------------- */
function svgEl(tag, attrs, parent) {
const e = document.createElementNS("http://www.w3.org/2000/svg", tag);
for (const k in attrs) e.setAttribute(k, attrs[k]);
@@ -135,12 +162,12 @@ function svgEl(tag, attrs, parent) {
return e;
}
/* ---------- scatter ---------- */
/* ---------- scatter -------------------------------------------------------- */
(function scatter() {
const box = el("scatter"); box.innerHTML = "";
const W = 520, H = 340, m = { l: 52, r: 16, t: 16, b: 44 };
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
const pts = scored.filter((x) => x.ii !== null);
const pts = scored.filter((x) => x.ii !== null && x.blend > 0);
const maxX = Math.max(...pts.map((p) => p.blend)) * 1.1 || 1, maxY = 60;
const X = (v) => m.l + (v / maxX) * (W - m.l - m.r);
const Y = (v) => H - m.b - (v / maxY) * (H - m.t - m.b);
@@ -158,21 +185,21 @@ function svgEl(tag, attrs, parent) {
const c = svgEl("circle", { cx: X(p.blend), cy: Y(p.ii), r: 5, fill: p.free ? "#2ecc71" : "#7cc4ff", "fill-opacity": 0.85, stroke: "#0b0d12", "stroke-width": 1.5 }, svg);
const lbl = svgEl("text", { x: X(p.blend) + 8, y: Y(p.ii) + 3, fill: "#c8cede", "font-size": 10 }, svg);
lbl.textContent = p.n.length > 22 ? p.n.slice(0, 21) + "…" : p.n;
c.onmousemove = (e) => showTip(`<b>${p.n}</b><br>Index ${p.ii.toFixed(1)}<br>${p.free ? "Free" : `$${p.pi} in / $${p.po} out per 1M`}`, e);
c.onmousemove = (e) => showTip(`<b>${p.n}</b><br>Index ${p.ii.toFixed(1)}<br>${p.free ? "Free" : `$${p.price.in} in / $${p.price.out} out per 1M`}`, e);
c.onmouseleave = hideTip;
});
})();
/* ---------- value ranking ---------- */
/* ---------- value ranking -------------------------------------------------- */
(function valueChart() {
const box = el("value"); box.innerHTML = "";
const W = 520, H = 340, m = { l: 96, r: 16, t: 12, b: 28 };
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
const pts = scored.filter((p) => !p.free).map((p) => ({ ...p, v: p.ii / p.blend })).sort((a, b) => b.v - a.v);
const pts = scored.filter((p) => !p.free && p.blend > 0).map((p) => ({ ...p, v: p.ii / p.blend })).sort((a, b) => b.v - a.v);
const maxV = Math.max(...pts.map((p) => p.v)) * 1.08;
const step = (H - m.t - m.b) / Math.max(pts.length, 1);
pts.forEach((p, i) => {
const y = 26 + i * step;
const y = 22 + i * step;
const w = Math.max(2, (p.v / maxV) * (W - m.l - m.r));
svgEl("rect", { x: m.l, y: y - 9, width: w, height: 15, rx: 3, fill: "#7cc4ff", "fill-opacity": 0.85 }, svg);
svgEl("text", { x: m.l - 8, y: y + 3, fill: "#c8cede", "font-size": 11, "text-anchor": "end" }, svg).textContent = p.n.length > 17 ? p.n.slice(0, 16) + "…" : p.n;
@@ -182,26 +209,25 @@ function svgEl(tag, attrs, parent) {
svgEl("text", { x: m.l, y: H - 6, fill: "#2ecc71", "font-size": 11 }, svg).textContent = "Free models (not ranked): " + free.map((f) => f.n).join(", ");
})();
/* ---------- model guide ---------- */
/* ---------- model guide ---------------------------------------------------- */
function modelCard(m) {
const tags = (m.best || []).map((b) => {
const key = Object.keys(CATEGORIES).find((k) => CATEGORIES[k].label.toLowerCase() === b.toLowerCase());
const key = b === "Free tier" ? "free" : b === "Image generation" ? "imagegen" : "";
const cls = key === "free" ? "tag free" : key === "imagegen" ? "tag warn" : "tag";
return `<span class="${cls}">${b}</span>`;
}).join("");
const ev = byId[m.id];
const idx = ev && ev.ii !== null ? `index ${ev.ii.toFixed(1)}` : "no public evals";
const idx = m.ii !== null ? `index ${m.ii.toFixed(1)}` : "no public evals";
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M context` : "context n/a";
return `<div class="card2">
<div class="hd"><h3>${m.name}</h3><span class="lab">${m.lab}</span></div>
<div class="meta">released ${m.released} · $${m.price.in}/${m.price.out} per 1M · ${m.ctx} context · ${m.modalities} · ${idx}</div>
<div class="hd"><h3>${m.n}</h3><span class="lab">${m.lab || "undisclosed"}</span></div>
<div class="meta">released ${m.released || "?"} · $${m.price.in}/${m.price.out} per 1M · ${ctx} · ${m.modalities} · ${idx}</div>
<div>${tags}</div>
<p class="blurb">${m.blurb}</p>
<p><b>Targeted for:</b> ${m.target}</p>
${m.target ? `<p><b>Targeted for:</b> ${m.target}</p>` : `<p><i>No write-up yet — this model was added to the catalogue automatically and needs a description.</i></p>`}
${m.caveats && m.caveats.length ? `<div class="caveats"><b>Caveats</b><ul>${m.caveats.map((c) => `<li>${c}</li>`).join("")}</ul></div>` : ""}
</div>`;
}
// loose keyword matching so a model's plain-language tags still hit the right filter
const FILTER_WORDS = {
agentic: ["agent"],
coding: ["cod", "repo"],
@@ -213,17 +239,17 @@ const FILTER_WORDS = {
};
function renderGuide(filter) {
const list = GO_MODELS.filter((m) => {
const list = MODELS.filter((m) => {
if (!filter || filter === "all") return true;
if (filter === "free") return m.free;
if (filter === "new") return m.released >= "2026-08-01";
if (filter === "new") return (m.released || "") >= "2026-08-01";
const words = FILTER_WORDS[filter] || [];
const hay = m.best.join(" ").toLowerCase();
return words.some((w) => hay.includes(w));
});
const extra = filter === "all" || filter === "free" ? `
<div class="note"><b>Also free outside the Go plan</b><br>${FREE_NOTES.map((f) => `<b>${f.name}</b> (${f.lab}, ${f.where}) — ${f.target} <i>${f.note}</i>`).join("<br>")}</div>` : "";
const none = list.length ? "" : `<div class="note">No model in the Go catalog matches this category. Every Go model is text-output — none of them generate images, so image generation needs a different provider.</div>`;
const none = list.length ? "" : `<div class="note">No model in the Go catalogue matches this category. Every Go model is text-output — none of them generate images, so image generation needs a different provider.</div>`;
el("guide").innerHTML = list.map(modelCard).join("") + extra + none;
}
@@ -249,7 +275,8 @@ el("foot").innerHTML = `
• Terminal-agent, tool-use, science and professional-work columns fall back to a second benchmark when a model has not been run on the first (Terminal-Bench 2.1 → 4.0 → Hard; τ²-bench → τ-bench Banking; GPQA → CritPt; GDPval → GDP-PDF). Scores are comparable within a column, but the underlying benchmark can differ per row — hover to check.<br>
• "Index" is the Artificial Analysis Intelligence Index, a weighted composite of those evaluations.<br>
• Prices are OpenCode Go list prices per 1M tokens (input/output). Blended price = 70% input / 30% output.<br>
• Free-tier models (MiniMax M3.1 Flash Free, Beam Free, LongCat 2.5 Preview Free) are limited-time offers with unlimited included usage.<br>
• Hy4 preview and LongCat 2.5 Preview Free have no public third-party evaluations yet, so they show as "n/a". Qwen3.8 Flash maps to Artificial Analysis' "Qwen3.8-Flash-Next" entry.<br>
• No model in the Go catalog generates images — all Go models are text-output. "Image understanding" rows mean the model can read images, video, PDFs or screenshots as input.<br><br>
Sources: <a href="https://artificialanalysis.ai">Artificial Analysis</a> (benchmarks, Intelligence Index), <a href="https://opencode.ai/data">OpenCode Data</a> (catalog, usage, context windows) and <a href="https://opencode.ai/docs/go">OpenCode Go docs</a> (pricing, allowances, privacy). Vendor announcements for model positioning. Retrieved 2026-10-21.`;
• Free-tier models are limited-time offers. Contributor-tier models (Gemini 4 Argon, Muse Spark) may use your prompts and completions for training.<br>
• Models marked "no write-up" were added to the catalogue automatically and do not yet have a hand-written description; models with no public third-party evaluation show "n/a" rather than borrowed numbers.<br>
• No model in the Go catalogue generates images — all are text-output. "Image understanding" means the model can read images, video, PDFs or screenshots as input.<br><br>
Sources: <a href="https://artificialanalysis.ai">Artificial Analysis</a> (benchmarks, Intelligence Index), <a href="https://models.dev">models.dev</a> (Go catalogue, pricing, context, modalities), <a href="https://opencode.ai/data">OpenCode Data</a> (usage and retention) and the <a href="https://opencode.ai/docs/go">OpenCode Go docs</a> (allowances, privacy). Vendor announcements for model positioning.<br>
Data snapshot: <b>${snapshotDate || "n/a"}</b> — refreshed weekly by the <code>weekly-eval.yml</code> workflow.`;