devadmin
4e5a13f7b6
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
283 lines
16 KiB
JavaScript
283 lines
16 KiB
JavaScript
import { CATALOG, SNAPSHOT } from "./catalog.js";
|
||
import { RAW } from "./evals.js";
|
||
import { GO_MODELS, FREE_NOTES } from "./data.js";
|
||
|
||
/* ---------- merge the three layers -----------------------------------------
|
||
catalog.js auto-generated — every model currently in OpenCode Go
|
||
evals.js auto-generated — benchmark scores per model
|
||
data.js hand-written — what each model is targeted for
|
||
--------------------------------------------------------------------------- */
|
||
const DESC = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));
|
||
const SCORES = Object.fromEntries(RAW.map((m) => [m.id, m]));
|
||
|
||
const MODELS = CATALOG.map((c) => {
|
||
const d = DESC[c.id] || {};
|
||
const s = SCORES[c.id] || {};
|
||
const m = {
|
||
...c,
|
||
best: d.best || [],
|
||
blurb: d.blurb || c.description || "No description published yet.",
|
||
target: d.target || "",
|
||
caveats: d.caveats || [],
|
||
written: !!d.blurb,
|
||
ii: s.ii ?? null,
|
||
tasks: {},
|
||
src: {}
|
||
};
|
||
m.blend = m.free ? 0 : 0.7 * m.price.in + 0.3 * m.price.out;
|
||
return m;
|
||
});
|
||
|
||
/* ---------- shared helpers ------------------------------------------------ */
|
||
const el = (id) => document.getElementById(id);
|
||
const tip = el("tip");
|
||
function showTip(html, e) {
|
||
tip.style.display = "block";
|
||
tip.style.left = Math.min(e.clientX + 14, window.innerWidth - 260) + "px";
|
||
tip.style.top = Math.min(e.clientY + 14, window.innerHeight - 140) + "px";
|
||
tip.innerHTML = html;
|
||
}
|
||
const hideTip = () => (tip.style.display = "none");
|
||
|
||
function color(v) {
|
||
if (v === null || v === undefined) return "transparent";
|
||
const stops = [
|
||
[14, 16, 26], [28, 50, 80], [31, 106, 143], [47, 163, 160],
|
||
[87, 204, 138], [182, 227, 106], [242, 214, 75], [243, 156, 61], [239, 83, 80]
|
||
];
|
||
const p = Math.max(0, Math.min(0.999, v / 100)) * (stops.length - 1);
|
||
const i = Math.floor(p), f = p - i, a = stops[i], b = stops[i + 1] || a;
|
||
const c = a.map((x, k) => Math.round(x + (b[k] - x) * f));
|
||
return `rgb(${c[0]},${c[1]},${c[2]})`;
|
||
}
|
||
const txtCol = (v) => (v >= 62 ? "#101319" : "#e6e9f2");
|
||
|
||
const TASKS = [
|
||
{ key: "coding", label: "Frontier coding", picks: [["fc", "FrontierCode"]] },
|
||
{ key: "scicode", label: "Sci coding", picks: [["scicode", "SciCode"]] },
|
||
{ key: "terminal", label: "Terminal agent", picks: [["tb21", "Terminal-Bench 2.1"], ["tb40", "Terminal-Bench 4.0"], ["tbh", "Terminal-Bench Hard"]] },
|
||
{ key: "tools", label: "Tool use (τ-bench)", picks: [["tau2", "τ²-bench"], ["taub", "τ-bench Banking"]] },
|
||
{ key: "agentic", label: "Agent workflows", picks: [["apex", "APEX-Agents"], ["analyst", "Analyst Agent"]] },
|
||
{ key: "auto", label: "Automation", picks: [["auto", "AutomationBench"]] },
|
||
{ key: "science", label: "Science reasoning", picks: [["gpqa", "GPQA Diamond"], ["critpt", "CritPt"]] },
|
||
{ key: "hle", label: "Knowledge (HLE)", picks: [["hle", "Humanity's Last Exam"]] },
|
||
{ key: "lcr", label: "Long context", picks: [["lcr", "AA Long-Context Reasoning"]] },
|
||
{ key: "ifbench", label: "Instruction following", picks: [["ifbench", "IFBench"]] },
|
||
{ key: "vision", label: "Vision (MMMU-Pro)", picks: [["mmmu", "MMMU-Pro"]] },
|
||
{ key: "pro", label: "Professional work", picks: [["gdpv", "GDPval-AA"], ["gdppdf", "GDP-PDF"]] }
|
||
];
|
||
|
||
MODELS.forEach((m) => {
|
||
TASKS.forEach((t) => {
|
||
for (const [f, label] of t.picks) {
|
||
const v = SCORES[m.id]?.[f];
|
||
if (v !== null && v !== undefined) { m.tasks[t.key] = v; m.src[t.key] = label; break; }
|
||
}
|
||
});
|
||
});
|
||
|
||
const scored = MODELS.filter((m) => m.ii !== null);
|
||
const topOverall = scored.slice().sort((a, b) => b.ii - a.ii)[0];
|
||
const bestValue = scored.filter((m) => !m.free).slice().sort((a, b) => b.ii / b.blend - a.ii / a.blend)[0];
|
||
const snapshotDate = (SNAPSHOT.generated || "").slice(0, 10);
|
||
|
||
/* ---------- header cards -------------------------------------------------- */
|
||
el("cards").innerHTML = [
|
||
["Models in Go", MODELS.length, "in the OpenCode Go catalogue"],
|
||
["With public evals", scored.length, "Artificial Analysis harness"],
|
||
["Top overall", topOverall.n, "index " + topOverall.ii.toFixed(1)],
|
||
["Best value", bestValue.n, "index " + bestValue.ii.toFixed(1) + " at $" + bestValue.blend.toFixed(2) + "/1M"],
|
||
["Free tier", MODELS.filter((m) => m.free).length, "limited-time free models"],
|
||
["Data snapshot", snapshotDate, "refreshed weekly by CI"]
|
||
].map(([k, v, s]) => `<div class="card"><div class="k">${k}</div><div class="v">${v}</div><div class="sub" style="font-size:11px">${s}</div></div>`).join("");
|
||
|
||
/* ---------- matrix --------------------------------------------------------- */
|
||
const cols = [
|
||
{ key: "name", label: "Model", get: (m) => m.n, txt: true },
|
||
{ key: "ii", label: "Index", get: (m) => m.ii },
|
||
{ key: "blend", label: "Price $/1M", get: (m) => m.blend, price: true },
|
||
...TASKS.map((t) => ({ key: t.key, label: t.label, get: (m) => m.tasks[t.key] }))
|
||
];
|
||
let sortKey = "ii", sortDir = -1;
|
||
|
||
function renderMatrix() {
|
||
const best = {};
|
||
cols.forEach((c) => {
|
||
if (c.key === "name" || c.key === "blend") return;
|
||
const vals = MODELS.map((m) => c.get(m)).filter((v) => v !== null && v !== undefined);
|
||
best[c.key] = vals.length ? Math.max(...vals) : null;
|
||
});
|
||
const rows = MODELS.slice().sort((a, b) => {
|
||
const c = cols.find((x) => x.key === sortKey);
|
||
const va = c.get(a), vb = c.get(b);
|
||
if (sortKey === "name") return String(va).localeCompare(String(vb)) * sortDir;
|
||
if (va === null || va === undefined) return 1;
|
||
if (vb === null || vb === undefined) return -1;
|
||
return (va - vb) * sortDir;
|
||
});
|
||
let h = "<thead><tr>";
|
||
cols.forEach((c) => {
|
||
h += `<th class="${c.txt ? "tl" : ""} ${c.key === sortKey ? "sorted" : ""}" data-k="${c.key}">${c.label}${c.key === sortKey ? (sortDir < 0 ? " ▾" : " ▴") : ""}</th>`;
|
||
});
|
||
h += "</tr></thead><tbody>";
|
||
rows.forEach((m) => {
|
||
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M ctx` : "ctx n/a";
|
||
h += `<tr><td class="name">${m.n}${m.free ? '<span class="tag free">free</span>' : ""}${!m.written ? '<span class="tag warn">no write-up</span>' : ""}<div class="price">${m.lab || "—"} · $${m.price.in}/${m.price.out} · ${ctx}${m.ii === null ? " · no public evals" : ""}</div></td>`;
|
||
cols.forEach((c) => {
|
||
if (c.key === "name") return;
|
||
const v = c.get(m);
|
||
if (c.key === "ii") {
|
||
h += `<td><div class="cell" style="background:${v === null ? "transparent" : "#1b2233"}">${v === null ? '<span class="na">n/a</span>' : `<b>${v.toFixed(1)}</b>`}</div></td>`;
|
||
return;
|
||
}
|
||
if (c.key === "blend") { h += `<td class="price">${m.free ? "$0" : "$" + v.toFixed(2)}</td>`; return; }
|
||
if (v === null || v === undefined) { h += `<td><div class="cell na">–</div></td>`; return; }
|
||
const isBest = best[c.key] !== null && Math.abs(v - best[c.key]) < 0.001;
|
||
h += `<td><div class="cell ${isBest ? "best" : ""}" style="background:${color(v)};color:${txtCol(v)}" data-m="${m.n}" data-t="${c.label}" data-src="${m.src[c.key]}">${v.toFixed(0)}<div class="bar" style="width:${v}%"></div></div></td>`;
|
||
});
|
||
h += "</tr>";
|
||
});
|
||
h += "</tbody>";
|
||
const t = el("tbl");
|
||
t.innerHTML = h;
|
||
t.querySelectorAll("th").forEach((th) => {
|
||
th.onclick = () => {
|
||
const k = th.dataset.k;
|
||
if (k === sortKey) sortDir *= -1; else { sortKey = k; sortDir = k === "name" ? 1 : -1; }
|
||
renderMatrix();
|
||
};
|
||
});
|
||
t.querySelectorAll(".cell[data-m]").forEach((c) => {
|
||
c.onmousemove = (e) => showTip(`<b>${c.dataset.m}</b><br>${c.dataset.t} — ${c.dataset.src}`, e);
|
||
c.onmouseleave = hideTip;
|
||
});
|
||
}
|
||
renderMatrix();
|
||
|
||
/* ---------- svg helpers ---------------------------------------------------- */
|
||
function svgEl(tag, attrs, parent) {
|
||
const e = document.createElementNS("http://www.w3.org/2000/svg", tag);
|
||
for (const k in attrs) e.setAttribute(k, attrs[k]);
|
||
if (parent) parent.appendChild(e);
|
||
return e;
|
||
}
|
||
|
||
/* ---------- scatter -------------------------------------------------------- */
|
||
(function scatter() {
|
||
const box = el("scatter"); box.innerHTML = "";
|
||
const W = 520, H = 340, m = { l: 52, r: 16, t: 16, b: 44 };
|
||
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
|
||
const pts = scored.filter((x) => x.ii !== null && x.blend > 0);
|
||
const maxX = Math.max(...pts.map((p) => p.blend)) * 1.1 || 1, maxY = 60;
|
||
const X = (v) => m.l + (v / maxX) * (W - m.l - m.r);
|
||
const Y = (v) => H - m.b - (v / maxY) * (H - m.t - m.b);
|
||
for (let g = 0; g <= maxY; g += 10) {
|
||
svgEl("line", { x1: m.l, y1: Y(g), x2: W - m.r, y2: Y(g), stroke: "#232838" }, svg);
|
||
svgEl("text", { x: m.l - 8, y: Y(g) + 4, fill: "#8b93a7", "font-size": 10, "text-anchor": "end" }, svg).textContent = g;
|
||
}
|
||
for (let g = 0; g <= maxX; g += 2) {
|
||
svgEl("line", { x1: X(g), y1: m.t, x2: X(g), y2: H - m.b, stroke: "#1c2130" }, svg);
|
||
svgEl("text", { x: X(g), y: H - m.b + 16, fill: "#8b93a7", "font-size": 10, "text-anchor": "middle" }, svg).textContent = "$" + g;
|
||
}
|
||
svgEl("text", { x: (m.l + W - m.r) / 2, y: H - 6, fill: "#8b93a7", "font-size": 11, "text-anchor": "middle" }, svg).textContent = "blended price $/1M tokens";
|
||
svgEl("text", { x: 14, y: (m.t + H - m.b) / 2, fill: "#8b93a7", "font-size": 11, "text-anchor": "middle", transform: `rotate(-90 14 ${(m.t + H - m.b) / 2})` }, svg).textContent = "Intelligence Index";
|
||
pts.forEach((p) => {
|
||
const c = svgEl("circle", { cx: X(p.blend), cy: Y(p.ii), r: 5, fill: p.free ? "#2ecc71" : "#7cc4ff", "fill-opacity": 0.85, stroke: "#0b0d12", "stroke-width": 1.5 }, svg);
|
||
const lbl = svgEl("text", { x: X(p.blend) + 8, y: Y(p.ii) + 3, fill: "#c8cede", "font-size": 10 }, svg);
|
||
lbl.textContent = p.n.length > 22 ? p.n.slice(0, 21) + "…" : p.n;
|
||
c.onmousemove = (e) => showTip(`<b>${p.n}</b><br>Index ${p.ii.toFixed(1)}<br>${p.free ? "Free" : `$${p.price.in} in / $${p.price.out} out per 1M`}`, e);
|
||
c.onmouseleave = hideTip;
|
||
});
|
||
})();
|
||
|
||
/* ---------- value ranking -------------------------------------------------- */
|
||
(function valueChart() {
|
||
const box = el("value"); box.innerHTML = "";
|
||
const W = 520, H = 340, m = { l: 96, r: 16, t: 12, b: 28 };
|
||
const svg = svgEl("svg", { viewBox: `0 0 ${W} ${H}`, width: "100%", height: "auto" }, box);
|
||
const pts = scored.filter((p) => !p.free && p.blend > 0).map((p) => ({ ...p, v: p.ii / p.blend })).sort((a, b) => b.v - a.v);
|
||
const maxV = Math.max(...pts.map((p) => p.v)) * 1.08;
|
||
const step = (H - m.t - m.b) / Math.max(pts.length, 1);
|
||
pts.forEach((p, i) => {
|
||
const y = 22 + i * step;
|
||
const w = Math.max(2, (p.v / maxV) * (W - m.l - m.r));
|
||
svgEl("rect", { x: m.l, y: y - 9, width: w, height: 15, rx: 3, fill: "#7cc4ff", "fill-opacity": 0.85 }, svg);
|
||
svgEl("text", { x: m.l - 8, y: y + 3, fill: "#c8cede", "font-size": 11, "text-anchor": "end" }, svg).textContent = p.n.length > 17 ? p.n.slice(0, 16) + "…" : p.n;
|
||
svgEl("text", { x: m.l + w + 6, y: y + 3, fill: "#8b93a7", "font-size": 10 }, svg).textContent = p.v.toFixed(0) + " pts/$";
|
||
});
|
||
const free = scored.filter((p) => p.free);
|
||
svgEl("text", { x: m.l, y: H - 6, fill: "#2ecc71", "font-size": 11 }, svg).textContent = "Free models (not ranked): " + free.map((f) => f.n).join(", ");
|
||
})();
|
||
|
||
/* ---------- model guide ---------------------------------------------------- */
|
||
function modelCard(m) {
|
||
const tags = (m.best || []).map((b) => {
|
||
const key = b === "Free tier" ? "free" : b === "Image generation" ? "imagegen" : "";
|
||
const cls = key === "free" ? "tag free" : key === "imagegen" ? "tag warn" : "tag";
|
||
return `<span class="${cls}">${b}</span>`;
|
||
}).join("");
|
||
const idx = m.ii !== null ? `index ${m.ii.toFixed(1)}` : "no public evals";
|
||
const ctx = m.ctx ? `${(m.ctx / 1e6).toFixed(m.ctx % 1e6 ? 2 : 0)}M context` : "context n/a";
|
||
return `<div class="card2">
|
||
<div class="hd"><h3>${m.n}</h3><span class="lab">${m.lab || "undisclosed"}</span></div>
|
||
<div class="meta">released ${m.released || "?"} · $${m.price.in}/${m.price.out} per 1M · ${ctx} · ${m.modalities} · ${idx}</div>
|
||
<div>${tags}</div>
|
||
<p class="blurb">${m.blurb}</p>
|
||
${m.target ? `<p><b>Targeted for:</b> ${m.target}</p>` : `<p><i>No write-up yet — this model was added to the catalogue automatically and needs a description.</i></p>`}
|
||
${m.caveats && m.caveats.length ? `<div class="caveats"><b>Caveats</b><ul>${m.caveats.map((c) => `<li>${c}</li>`).join("")}</ul></div>` : ""}
|
||
</div>`;
|
||
}
|
||
|
||
const FILTER_WORDS = {
|
||
agentic: ["agent"],
|
||
coding: ["cod", "repo"],
|
||
data: ["data", "spreadsheet", "analys"],
|
||
vision: ["vision", "image", "multimodal", "perception", "visual"],
|
||
imagegen: ["generation", "generat"],
|
||
research: ["research", "knowledge work", "document", "deep"],
|
||
fast: ["cheap", "fast", "high-volume", "efficient", "budget"]
|
||
};
|
||
|
||
function renderGuide(filter) {
|
||
const list = MODELS.filter((m) => {
|
||
if (!filter || filter === "all") return true;
|
||
if (filter === "free") return m.free;
|
||
if (filter === "new") return (m.released || "") >= "2026-08-01";
|
||
const words = FILTER_WORDS[filter] || [];
|
||
const hay = m.best.join(" ").toLowerCase();
|
||
return words.some((w) => hay.includes(w));
|
||
});
|
||
const extra = filter === "all" || filter === "free" ? `
|
||
<div class="note"><b>Also free outside the Go plan</b><br>${FREE_NOTES.map((f) => `<b>${f.name}</b> (${f.lab}, ${f.where}) — ${f.target} <i>${f.note}</i>`).join("<br>")}</div>` : "";
|
||
const none = list.length ? "" : `<div class="note">No model in the Go catalogue matches this category. Every Go model is text-output — none of them generate images, so image generation needs a different provider.</div>`;
|
||
el("guide").innerHTML = list.map(modelCard).join("") + extra + none;
|
||
}
|
||
|
||
const filters = [
|
||
["all", "All models"], ["free", "Free tier"], ["new", "New (since Aug 2026)"],
|
||
["agentic", "Agentic"], ["coding", "Coding"], ["data", "Data analysis"],
|
||
["vision", "Image understanding"], ["imagegen", "Image generation"],
|
||
["research", "Deep research"], ["fast", "Fast & cheap"]
|
||
];
|
||
el("filters").innerHTML = filters.map(([k, l]) => `<div class="chip${k === "all" ? " on" : ""}" data-f="${k}">${l}</div>`).join("");
|
||
el("filters").querySelectorAll(".chip").forEach((c) => {
|
||
c.onclick = () => {
|
||
el("filters").querySelectorAll(".chip").forEach((x) => x.classList.remove("on"));
|
||
c.classList.add("on");
|
||
renderGuide(c.dataset.f);
|
||
};
|
||
});
|
||
renderGuide("all");
|
||
|
||
el("foot").innerHTML = `
|
||
<b>How to read the matrix</b><br>
|
||
• Each cell is the highest-scoring published evaluation for that task family on a 0–100 scale; the benchmark behind it is named in the hover tooltip.<br>
|
||
• Terminal-agent, tool-use, science and professional-work columns fall back to a second benchmark when a model has not been run on the first (Terminal-Bench 2.1 → 4.0 → Hard; τ²-bench → τ-bench Banking; GPQA → CritPt; GDPval → GDP-PDF). Scores are comparable within a column, but the underlying benchmark can differ per row — hover to check.<br>
|
||
• "Index" is the Artificial Analysis Intelligence Index, a weighted composite of those evaluations.<br>
|
||
• Prices are OpenCode Go list prices per 1M tokens (input/output). Blended price = 70% input / 30% output.<br>
|
||
• Free-tier models are limited-time offers. Contributor-tier models (Gemini 4 Argon, Muse Spark) may use your prompts and completions for training.<br>
|
||
• Models marked "no write-up" were added to the catalogue automatically and do not yet have a hand-written description; models with no public third-party evaluation show "n/a" rather than borrowed numbers.<br>
|
||
• No model in the Go catalogue generates images — all are text-output. "Image understanding" means the model can read images, video, PDFs or screenshots as input.<br><br>
|
||
Sources: <a href="https://artificialanalysis.ai">Artificial Analysis</a> (benchmarks, Intelligence Index), <a href="https://models.dev">models.dev</a> (Go catalogue, pricing, context, modalities), <a href="https://opencode.ai/data">OpenCode Data</a> (usage and retention) and the <a href="https://opencode.ai/docs/go">OpenCode Go docs</a> (allowances, privacy). Vendor announcements for model positioning.<br>
|
||
Data snapshot: <b>${snapshotDate || "n/a"}</b> — refreshed weekly by the <code>weekly-eval.yml</code> workflow.`;
|