Automate the evaluation on a weekly schedule
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
This commit is contained in:
+65
-38
@@ -1,7 +1,34 @@
|
||||
import { CATALOG, SNAPSHOT } from "./catalog.js";
|
||||
import { RAW } from "./evals.js";
|
||||
import { GO_MODELS, FREE_NOTES, CATEGORIES } from "./data.js";
|
||||
import { GO_MODELS, FREE_NOTES } from "./data.js";
|
||||
|
||||
/* ---------- shared helpers ---------- */
|
||||
/* ---------- merge the three layers -----------------------------------------
|
||||
catalog.js auto-generated — every model currently in OpenCode Go
|
||||
evals.js auto-generated — benchmark scores per model
|
||||
data.js hand-written — what each model is targeted for
|
||||
--------------------------------------------------------------------------- */
|
||||
const DESC = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));
|
||||
const SCORES = Object.fromEntries(RAW.map((m) => [m.id, m]));
|
||||
|
||||
const MODELS = CATALOG.map((c) => {
|
||||
const d = DESC[c.id] || {};
|
||||
const s = SCORES[c.id] || {};
|
||||
const m = {
|
||||
...c,
|
||||
best: d.best || [],
|
||||
blurb: d.blurb || c.description || "No description published yet.",
|
||||
target: d.target || "",
|
||||
caveats: d.caveats || [],
|
||||
written: !!d.blurb,
|
||||
ii: s.ii ?? null,
|
||||
tasks: {},
|
||||
src: {}
|
||||
};
|
||||
m.blend = m.free ? 0 : 0.7 * m.price.in + 0.3 * m.price.out;
|
||||
return m;
|
||||
});
|
||||
|
||||
/* ---------- shared helpers ------------------------------------------------ */
|
||||
const el = (id) => document.getElementById(id);
|
||||
const tip = el("tip");
|
||||
function showTip(html, e) {
|
||||
@@ -40,32 +67,31 @@ const TASKS = [
|
||||
{ key: "pro", label: "Professional work", picks: [["gdpv", "GDPval-AA"], ["gdppdf", "GDP-PDF"]] }
|
||||
];
|
||||
|
||||
RAW.forEach((m) => {
|
||||
m.tasks = {}; m.src = {};
|
||||
MODELS.forEach((m) => {
|
||||
TASKS.forEach((t) => {
|
||||
for (const [f, label] of t.picks) {
|
||||
const v = m[f];
|
||||
const v = SCORES[m.id]?.[f];
|
||||
if (v !== null && v !== undefined) { m.tasks[t.key] = v; m.src[t.key] = label; break; }
|
||||
}
|
||||
});
|
||||
m.blend = m.free ? 0 : 0.7 * m.pi + 0.3 * m.po;
|
||||
});
|
||||
|
||||
const byId = Object.fromEntries(RAW.map((m) => [m.id, m]));
|
||||
|
||||
/* ---------- header cards ---------- */
|
||||
const scored = RAW.filter((m) => m.ii !== null);
|
||||
const scored = MODELS.filter((m) => m.ii !== null);
|
||||
const topOverall = scored.slice().sort((a, b) => b.ii - a.ii)[0];
|
||||
const bestValue = scored.filter((m) => !m.free).slice().sort((a, b) => b.ii / b.blend - a.ii / a.blend)[0];
|
||||
const snapshotDate = (SNAPSHOT.generated || "").slice(0, 10);
|
||||
|
||||
/* ---------- header cards -------------------------------------------------- */
|
||||
el("cards").innerHTML = [
|
||||
["Models in Go", RAW.length, "in the Go catalog"],
|
||||
["Models in Go", MODELS.length, "in the OpenCode Go catalogue"],
|
||||
["With public evals", scored.length, "Artificial Analysis harness"],
|
||||
["Top overall", topOverall.n, "index " + topOverall.ii.toFixed(1)],
|
||||
["Best value", bestValue.n, "index " + bestValue.ii.toFixed(1) + " at $" + bestValue.blend.toFixed(2) + "/1M"],
|
||||
["Free tier", RAW.filter((m) => m.free).length, "limited-time free models"]
|
||||
["Free tier", MODELS.filter((m) => m.free).length, "limited-time free models"],
|
||||
["Data snapshot", snapshotDate, "refreshed weekly by CI"]
|
||||
].map(([k, v, s]) => `<div class="card"><div class="k">${k}</div><div class="v">${v}</div><div class="sub" style="font-size:11px">${s}</div></div>`).join("");
|
||||
|
||||
/* ---------- matrix ---------- */
|
||||
/* ---------- matrix --------------------------------------------------------- */
|
||||
const cols = [
|
||||
{ key: "name", label: "Model", get: (m) => m.n, txt: true },
|
||||
{ key: "ii", label: "Index", get: (m) => m.ii },
|
||||
@@ -78,10 +104,10 @@ function renderMatrix() {
|
||||
const best = {};
|
||||
cols.forEach((c) => {
|
||||
if (c.key === "name" || c.key === "blend") return;
|
||||
const vals = RAW.map((m) => c.get(m)).filter((v) => v !== null && v !== undefined);
|
||||
const vals = MODELS.map((m) => c.get(m)).filter((v) => v !== null && v !== undefined);
|
||||
best[c.key] = vals.length ? Math.max(...vals) : null;
|
||||
});
|
||||
const rows = RAW.slice().sort((a, b) => {
|
||||
const rows = MODELS.slice().sort((a, b) => {
|
||||
const c = cols.find((x) => x.key === sortKey);
|
||||
const va = c.get(a), vb = c.get(b);
|
||||
if (sortKey === "name") return String(va).localeCompare(String(vb)) * sortDir;
|
||||
@@ -95,7 +121,8 @@ function renderMatrix() {
|
||||
});
|
||||
h += "</tr></thead><tbody>";
|
||||
rows.forEach((m) => {
|
||||
h += `<tr><td class="name">${m.n}${m.free ? '<span class="tag free">free</span>' : ""}<div class="price">${m.lab} · $${m.pi}/${m.po} · ${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M ctx${m.ii === null ? " · no public evals" : ""}</div></td>`;
|
||||
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M ctx` : "ctx n/a";
|
||||
h += `<tr><td class="name">${m.n}${m.free ? '<span class="tag free">free</span>' : ""}${!m.written ? '<span class="tag warn">no write-up</span>' : ""}<div class="price">${m.lab || "—"} · $${m.price.in}/${m.price.out} · ${ctx}${m.ii === null ? " · no public evals" : ""}</div></td>`;
|
||||
cols.forEach((c) => {
|
||||
if (c.key === "name") return;
|
||||
const v = c.get(m);
|
||||
@@ -127,7 +154,7 @@ function renderMatrix() {
|
||||
}
|
||||
renderMatrix();
|
||||
|
||||
/* ---------- svg helpers ---------- */
|
||||
/* ---------- svg helpers ---------------------------------------------------- */
|
||||
function svgEl(tag, attrs, parent) {
|
||||
const e = document.createElementNS("http://www.w3.org/2000/svg", tag);
|
||||
for (const k in attrs) e.setAttribute(k, attrs[k]);
|
||||
@@ -135,12 +162,12 @@ function svgEl(tag, attrs, parent) {
|
||||
return e;
|
||||
}
|
||||
|
||||
/* ---------- scatter ---------- */
|
||||
/* ---------- scatter -------------------------------------------------------- */
|
||||
(function scatter() {
|
||||
const box = el("scatter"); box.innerHTML = "";
|
||||
const W = 520, H = 340, m = { l: 52, r: 16, t: 16, b: 44 };
|
||||
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
|
||||
const pts = scored.filter((x) => x.ii !== null);
|
||||
const pts = scored.filter((x) => x.ii !== null && x.blend > 0);
|
||||
const maxX = Math.max(...pts.map((p) => p.blend)) * 1.1 || 1, maxY = 60;
|
||||
const X = (v) => m.l + (v / maxX) * (W - m.l - m.r);
|
||||
const Y = (v) => H - m.b - (v / maxY) * (H - m.t - m.b);
|
||||
@@ -158,21 +185,21 @@ function svgEl(tag, attrs, parent) {
|
||||
const c = svgEl("circle", { cx: X(p.blend), cy: Y(p.ii), r: 5, fill: p.free ? "#2ecc71" : "#7cc4ff", "fill-opacity": 0.85, stroke: "#0b0d12", "stroke-width": 1.5 }, svg);
|
||||
const lbl = svgEl("text", { x: X(p.blend) + 8, y: Y(p.ii) + 3, fill: "#c8cede", "font-size": 10 }, svg);
|
||||
lbl.textContent = p.n.length > 22 ? p.n.slice(0, 21) + "…" : p.n;
|
||||
c.onmousemove = (e) => showTip(`<b>${p.n}</b><br>Index ${p.ii.toFixed(1)}<br>${p.free ? "Free" : `$${p.pi} in / $${p.po} out per 1M`}`, e);
|
||||
c.onmousemove = (e) => showTip(`<b>${p.n}</b><br>Index ${p.ii.toFixed(1)}<br>${p.free ? "Free" : `$${p.price.in} in / $${p.price.out} out per 1M`}`, e);
|
||||
c.onmouseleave = hideTip;
|
||||
});
|
||||
})();
|
||||
|
||||
/* ---------- value ranking ---------- */
|
||||
/* ---------- value ranking -------------------------------------------------- */
|
||||
(function valueChart() {
|
||||
const box = el("value"); box.innerHTML = "";
|
||||
const W = 520, H = 340, m = { l: 96, r: 16, t: 12, b: 28 };
|
||||
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
|
||||
const pts = scored.filter((p) => !p.free).map((p) => ({ ...p, v: p.ii / p.blend })).sort((a, b) => b.v - a.v);
|
||||
const pts = scored.filter((p) => !p.free && p.blend > 0).map((p) => ({ ...p, v: p.ii / p.blend })).sort((a, b) => b.v - a.v);
|
||||
const maxV = Math.max(...pts.map((p) => p.v)) * 1.08;
|
||||
const step = (H - m.t - m.b) / Math.max(pts.length, 1);
|
||||
pts.forEach((p, i) => {
|
||||
const y = 26 + i * step;
|
||||
const y = 22 + i * step;
|
||||
const w = Math.max(2, (p.v / maxV) * (W - m.l - m.r));
|
||||
svgEl("rect", { x: m.l, y: y - 9, width: w, height: 15, rx: 3, fill: "#7cc4ff", "fill-opacity": 0.85 }, svg);
|
||||
svgEl("text", { x: m.l - 8, y: y + 3, fill: "#c8cede", "font-size": 11, "text-anchor": "end" }, svg).textContent = p.n.length > 17 ? p.n.slice(0, 16) + "…" : p.n;
|
||||
@@ -182,26 +209,25 @@ function svgEl(tag, attrs, parent) {
|
||||
svgEl("text", { x: m.l, y: H - 6, fill: "#2ecc71", "font-size": 11 }, svg).textContent = "Free models (not ranked): " + free.map((f) => f.n).join(", ");
|
||||
})();
|
||||
|
||||
/* ---------- model guide ---------- */
|
||||
/* ---------- model guide ---------------------------------------------------- */
|
||||
function modelCard(m) {
|
||||
const tags = (m.best || []).map((b) => {
|
||||
const key = Object.keys(CATEGORIES).find((k) => CATEGORIES[k].label.toLowerCase() === b.toLowerCase());
|
||||
const key = b === "Free tier" ? "free" : b === "Image generation" ? "imagegen" : "";
|
||||
const cls = key === "free" ? "tag free" : key === "imagegen" ? "tag warn" : "tag";
|
||||
return `<span class="${cls}">${b}</span>`;
|
||||
}).join("");
|
||||
const ev = byId[m.id];
|
||||
const idx = ev && ev.ii !== null ? `index ${ev.ii.toFixed(1)}` : "no public evals";
|
||||
const idx = m.ii !== null ? `index ${m.ii.toFixed(1)}` : "no public evals";
|
||||
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M context` : "context n/a";
|
||||
return `<div class="card2">
|
||||
<div class="hd"><h3>${m.name}</h3><span class="lab">${m.lab}</span></div>
|
||||
<div class="meta">released ${m.released} · $${m.price.in}/${m.price.out} per 1M · ${m.ctx} context · ${m.modalities} · ${idx}</div>
|
||||
<div class="hd"><h3>${m.n}</h3><span class="lab">${m.lab || "undisclosed"}</span></div>
|
||||
<div class="meta">released ${m.released || "?"} · $${m.price.in}/${m.price.out} per 1M · ${ctx} · ${m.modalities} · ${idx}</div>
|
||||
<div>${tags}</div>
|
||||
<p class="blurb">${m.blurb}</p>
|
||||
<p><b>Targeted for:</b> ${m.target}</p>
|
||||
${m.target ? `<p><b>Targeted for:</b> ${m.target}</p>` : `<p><i>No write-up yet — this model was added to the catalogue automatically and needs a description.</i></p>`}
|
||||
${m.caveats && m.caveats.length ? `<div class="caveats"><b>Caveats</b><ul>${m.caveats.map((c) => `<li>${c}</li>`).join("")}</ul></div>` : ""}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// loose keyword matching so a model's plain-language tags still hit the right filter
|
||||
const FILTER_WORDS = {
|
||||
agentic: ["agent"],
|
||||
coding: ["cod", "repo"],
|
||||
@@ -213,17 +239,17 @@ const FILTER_WORDS = {
|
||||
};
|
||||
|
||||
function renderGuide(filter) {
|
||||
const list = GO_MODELS.filter((m) => {
|
||||
const list = MODELS.filter((m) => {
|
||||
if (!filter || filter === "all") return true;
|
||||
if (filter === "free") return m.free;
|
||||
if (filter === "new") return m.released >= "2026-08-01";
|
||||
if (filter === "new") return (m.released || "") >= "2026-08-01";
|
||||
const words = FILTER_WORDS[filter] || [];
|
||||
const hay = m.best.join(" ").toLowerCase();
|
||||
return words.some((w) => hay.includes(w));
|
||||
});
|
||||
const extra = filter === "all" || filter === "free" ? `
|
||||
<div class="note"><b>Also free outside the Go plan</b><br>${FREE_NOTES.map((f) => `<b>${f.name}</b> (${f.lab}, ${f.where}) — ${f.target} <i>${f.note}</i>`).join("<br>")}</div>` : "";
|
||||
const none = list.length ? "" : `<div class="note">No model in the Go catalog matches this category. Every Go model is text-output — none of them generate images, so image generation needs a different provider.</div>`;
|
||||
const none = list.length ? "" : `<div class="note">No model in the Go catalogue matches this category. Every Go model is text-output — none of them generate images, so image generation needs a different provider.</div>`;
|
||||
el("guide").innerHTML = list.map(modelCard).join("") + extra + none;
|
||||
}
|
||||
|
||||
@@ -249,7 +275,8 @@ el("foot").innerHTML = `
|
||||
• Terminal-agent, tool-use, science and professional-work columns fall back to a second benchmark when a model has not been run on the first (Terminal-Bench 2.1 → 4.0 → Hard; τ²-bench → τ-bench Banking; GPQA → CritPt; GDPval → GDP-PDF). Scores are comparable within a column, but the underlying benchmark can differ per row — hover to check.<br>
|
||||
• "Index" is the Artificial Analysis Intelligence Index, a weighted composite of those evaluations.<br>
|
||||
• Prices are OpenCode Go list prices per 1M tokens (input/output). Blended price = 70% input / 30% output.<br>
|
||||
• Free-tier models (MiniMax M3.1 Flash Free, Beam Free, LongCat 2.5 Preview Free) are limited-time offers with unlimited included usage.<br>
|
||||
• Hy4 preview and LongCat 2.5 Preview Free have no public third-party evaluations yet, so they show as "n/a". Qwen3.8 Flash maps to Artificial Analysis' "Qwen3.8-Flash-Next" entry.<br>
|
||||
• No model in the Go catalog generates images — all Go models are text-output. "Image understanding" rows mean the model can read images, video, PDFs or screenshots as input.<br><br>
|
||||
Sources: <a href="https://artificialanalysis.ai">Artificial Analysis</a> (benchmarks, Intelligence Index), <a href="https://opencode.ai/data">OpenCode Data</a> (catalog, usage, context windows) and <a href="https://opencode.ai/docs/go">OpenCode Go docs</a> (pricing, allowances, privacy). Vendor announcements for model positioning. Retrieved 2026-10-21.`;
|
||||
• Free-tier models are limited-time offers. Contributor-tier models (Gemini 4 Argon, Muse Spark) may use your prompts and completions for training.<br>
|
||||
• Models marked "no write-up" were added to the catalogue automatically and do not yet have a hand-written description; models with no public third-party evaluation show "n/a" rather than borrowed numbers.<br>
|
||||
• No model in the Go catalogue generates images — all are text-output. "Image understanding" means the model can read images, video, PDFs or screenshots as input.<br><br>
|
||||
Sources: <a href="https://artificialanalysis.ai">Artificial Analysis</a> (benchmarks, Intelligence Index), <a href="https://models.dev">models.dev</a> (Go catalogue, pricing, context, modalities), <a href="https://opencode.ai/data">OpenCode Data</a> (usage and retention) and the <a href="https://opencode.ai/docs/go">OpenCode Go docs</a> (allowances, privacy). Vendor announcements for model positioning.<br>
|
||||
Data snapshot: <b>${snapshotDate || "n/a"}</b> — refreshed weekly by the <code>weekly-eval.yml</code> workflow.`;
|
||||
|
||||
Reference in New Issue
Block a user