Automate the evaluation on a weekly schedule
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env node
|
||||
// Regenerates MODELS.md from the three data layers.
|
||||
//
|
||||
// node scripts/build-models-md.mjs
|
||||
//
|
||||
// site/assets/catalog.js auto — catalogue metadata
|
||||
// site/assets/evals.js auto — benchmark scores
|
||||
// site/assets/data.js hand — descriptions and targeting
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const { CATALOG, SNAPSHOT } = await import(resolve(ROOT, "site/assets/catalog.js"));
|
||||
const { RAW } = await import(resolve(ROOT, "site/assets/evals.js"));
|
||||
const { GO_MODELS, FREE_NOTES } = await import(resolve(ROOT, "site/assets/data.js"));
|
||||
|
||||
const DESC = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));
|
||||
const SCORE = Object.fromEntries(RAW.map((m) => [m.id, m]));
|
||||
const fmtCtx = (c) => (c ? `${(c / 1e6).toFixed(c % 1e6 ? 2 : 0)}M` : "n/a");
|
||||
|
||||
let md = `# OpenCode Go — what each model is for
|
||||
|
||||
Short, practical notes on every model in the OpenCode Go catalogue: what each one is targeted for
|
||||
(agentic work, coding, data analysis, image understanding, deep research, high-volume chores),
|
||||
what it costs, and what to watch out for.
|
||||
|
||||
Evaluated scores come from Artificial Analysis (single consistent harness) plus vendor model cards.
|
||||
Catalogue metadata comes from models.dev's OpenCode Go provider. Data snapshot: **${(SNAPSHOT.generated || "").slice(0, 10)}**.
|
||||
The interactive version with the full task matrix lives at \`site/index.html\`.
|
||||
|
||||
> No model in the Go catalogue generates images. Every Go model is text-output; several accept image,
|
||||
> video, PDF or audio as **input**.
|
||||
|
||||
## Quick picks
|
||||
|
||||
| If you need… | Use |
|
||||
|---|---|
|
||||
| The strongest model overall, for hard agentic + coding work | Gemini 4 Argon Contributor |
|
||||
| The best value per token for everyday coding agents | DeepSeek V4.1 Flash |
|
||||
| Zero-cost exploration (limited time) | MiniMax M3.1 Flash Free, Beam Free, LongCat 2.5 Preview Free |
|
||||
| Long-horizon agent runs with multimodal input | Muse Spark 1.3 Contributor, MiMo-V2.6-Pro |
|
||||
| Repo-scale code changes | Kimi K2.7 Code, GLM-5.3, Qwen3.8 Max |
|
||||
| Spreadsheets / data analysis deliverables | Kimi K3, Qwen3.8 Max |
|
||||
| Reading screenshots, charts and documents | Qwen3.8 Flash, GLM-5.3-Flash, MiniMax M3 |
|
||||
| High-volume, latency-sensitive work | Claude Haiku 5.5, GPT-6 Luna, MiMo-V2.5 |
|
||||
|
||||
## The catalogue
|
||||
|
||||
`;
|
||||
|
||||
const sorted = CATALOG.slice().sort((a, b) => {
|
||||
const ia = SCORE[a.id]?.ii ?? -1;
|
||||
const ib = SCORE[b.id]?.ii ?? -1;
|
||||
return ib - ia;
|
||||
});
|
||||
|
||||
for (const m of sorted) {
|
||||
const d = DESC[m.id];
|
||||
const s = SCORE[m.id];
|
||||
md += `### ${m.name}\n\n`;
|
||||
md += `**${m.lab || "undisclosed"}** · released ${m.released || "?"} · $${m.price.in} in / $${m.price.out} out per 1M tokens`;
|
||||
md += m.free ? " · **free tier (limited time)**" : "";
|
||||
md += ` · ${fmtCtx(m.ctx)} context · ${m.modalities}\n\n`;
|
||||
if (d?.best?.length) md += `Targeted for: **${d.best.join(", ")}**\n\n`;
|
||||
if (d?.blurb) md += `${d.blurb}\n\n`;
|
||||
else md += `${m.description || "_No description published._"}\n\n`;
|
||||
if (d?.target) md += `${d.target}\n\n`;
|
||||
else md += "_No hand-written write-up yet._\n\n";
|
||||
if (s?.ii !== null && s?.ii !== undefined) {
|
||||
md += `*Evaluated:* Intelligence Index ${s.ii}`;
|
||||
const extra = [
|
||||
s.fc != null ? `FrontierCode ${s.fc}` : null,
|
||||
s.tb21 != null ? `TB2.1 ${s.tb21}` : s.tb40 != null ? `TB4.0 ${s.tb40}` : null,
|
||||
s.auto != null ? `AutomationBench ${s.auto}` : null,
|
||||
s.gpqa != null ? `GPQA ${s.gpqa}` : null,
|
||||
s.hle != null ? `HLE ${s.hle}` : null,
|
||||
s.lcr != null ? `LCR ${s.lcr}` : null,
|
||||
s.mmmu != null ? `MMMU-Pro ${s.mmmu}` : null
|
||||
].filter(Boolean);
|
||||
if (extra.length) md += " · " + extra.join(" · ");
|
||||
md += "\n\n";
|
||||
} else {
|
||||
md += "*Evaluated:* no public third-party evaluation published yet.\n\n";
|
||||
}
|
||||
if (d?.caveats?.length) md += `**Caveats**\n\n${d.caveats.map((c) => `- ${c}`).join("\n")}\n\n`;
|
||||
}
|
||||
|
||||
md += `## Free models outside the Go plan\n\n`;
|
||||
for (const f of FREE_NOTES) {
|
||||
md += `### ${f.name} (${f.lab})\n\n`;
|
||||
md += `*Where:* ${f.where}\n\n${f.target}\n\n_${f.note}_\n\n`;
|
||||
}
|
||||
|
||||
md += `## Sources\n\n`;
|
||||
md += `- [OpenCode Go docs](https://opencode.ai/docs/go) — pricing, allowances, privacy and retention per model\n`;
|
||||
md += `- [models.dev](https://models.dev) — OpenCode Go catalogue metadata (pricing, context, modalities)\n`;
|
||||
md += `- [OpenCode Data](https://opencode.ai/data) — usage, retention, market share\n`;
|
||||
md += `- [Artificial Analysis](https://artificialanalysis.ai) — benchmark scores and the Intelligence Index\n`;
|
||||
md += `- Vendor announcements: Google (Gemini 4 Argon), Reflection AI (Beam), Meituan (LongCat), MiniMax, xAI (Grok 4.7), StepFun (Step 5), Qwen, Zhipu, Moonshot, DeepSeek, Xiaomi, Tencent\n`;
|
||||
|
||||
writeFileSync(resolve(ROOT, "MODELS.md"), md);
|
||||
console.log(`wrote MODELS.md (${md.length} bytes, ${sorted.length} models)`);
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env node
|
||||
// Opens a Gitea issue when the weekly refresh finds models that need a write-up.
|
||||
//
|
||||
// node scripts/notify-new-models.mjs
|
||||
//
|
||||
// Reads site/assets/snapshot.json (written by sync-catalog.mjs) and, if the catalogue
|
||||
// gained models, files an issue listing them so somebody writes the "targeted for"
|
||||
// descriptions in site/assets/data.js.
|
||||
//
|
||||
// Uses GITHUB_API_URL / GITEA_TOKEN (or GITHUB_TOKEN) from the environment.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const snapshot = JSON.parse(readFileSync(resolve(ROOT, "site/assets/snapshot.json"), "utf8"));
|
||||
|
||||
const api = process.env.GITHUB_API_URL || "https://git.wallacenet.ca/api/v1";
|
||||
const repo = process.env.GITHUB_REPOSITORY || "devadmin/OpenCodeGoEvals";
|
||||
const token = process.env.GITEA_TOKEN || process.env.GITHUB_TOKEN;
|
||||
|
||||
if (!snapshot.newModels?.length) {
|
||||
console.log("no new models — nothing to report");
|
||||
if (snapshot.deltas?.length) {
|
||||
console.log("score changes since last refresh:");
|
||||
for (const d of snapshot.deltas) console.log(` ${d.id}: ${d.from} → ${d.to}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const { CATALOG } = await import(resolve(ROOT, "site/assets/catalog.js"));
|
||||
const rows = snapshot.newModels
|
||||
.map((id) => CATALOG.find((c) => c.id === id))
|
||||
.filter(Boolean)
|
||||
.map(
|
||||
(c) =>
|
||||
`| ${c.name} | ${c.lab || "undisclosed"} | $${c.price.in}/${c.price.out} | ${c.ctx ? (c.ctx / 1e6).toFixed(0) + "M" : "n/a"} | ${c.description || "—"} |`
|
||||
);
|
||||
|
||||
const body = [
|
||||
"The weekly catalogue refresh found model(s) that are live in OpenCode Go but have no",
|
||||
"hand-written write-up yet.",
|
||||
"",
|
||||
"| Model | Lab | Price $/1M | Context | Vendor description |",
|
||||
"|---|---|---|---|---|",
|
||||
...rows,
|
||||
"",
|
||||
"To add them:",
|
||||
"",
|
||||
"1. Add an entry to `site/assets/data.js` under `GO_MODELS` with `id`, `best` (targeting tags),",
|
||||
" `blurb`, `target` and `caveats`.",
|
||||
"2. Run `node scripts/build-models-md.mjs` to regenerate `MODELS.md`.",
|
||||
"3. Commit — CI publishes the site.",
|
||||
"",
|
||||
`Detected ${new Date().toISOString().slice(0, 10)}.`
|
||||
].join("\n");
|
||||
|
||||
if (!token) {
|
||||
console.log("no token in environment — would file an issue with:\n");
|
||||
console.log(body);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const res = await fetch(`${api}/repos/${repo}/issues`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `token ${token}`,
|
||||
"User-Agent": "opencode-go-evals-bot"
|
||||
},
|
||||
body: JSON.stringify({ title: `New model(s) need write-ups: ${snapshot.newModels.join(", ")}`, body })
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
console.error(`could not create issue: ${res.status} ${await res.text()}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const issue = await res.json();
|
||||
console.log(`opened issue #${issue.number}: ${issue.html_url}`);
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Regenerates site/assets/evals.js from live sources.
|
||||
//
|
||||
// node scripts/refresh-evals.mjs
|
||||
//
|
||||
// Sources
|
||||
// - Artificial Analysis model pages (https://artificialanalysis.ai/models/<slug>)
|
||||
// The benchmark payload is embedded in the page as React Server Component flight data.
|
||||
// - OpenCode Data model JSON (https://opencode.ai/data/<lab>/<model>.json)
|
||||
// for context windows and OpenCode Go list prices.
|
||||
//
|
||||
// Usage of the AI slug differs slightly from the OpenCode Go model id, so they are
|
||||
// mapped explicitly below.
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { GO_MODELS } from "../site/assets/data.js";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";
|
||||
|
||||
// go model id -> { aa: artificial analysis slug, od: opencode data path }
|
||||
const MODELS = {
|
||||
"minimax-m3.1-flash-free": { aa: "minimax-m3-1-flash", od: "minimax/minimax-m3-1-flash" },
|
||||
"gemini-4-argon-contributor": { aa: "gemini-4-argon", od: "google/gemini-4-argon" },
|
||||
"claude-haiku-5-5": { aa: "claude-haiku-5-5", od: "anthropic/claude-haiku-5-5" },
|
||||
"beam-free": { aa: "reflection-beam", od: "reflection/beam" },
|
||||
// No Artificial Analysis entry for these two yet — they stay empty rather than
|
||||
// borrowing a different model's numbers (LongCat 2.0 / Hy3 are not LongCat 2.5 / Hy4).
|
||||
"longcat-2.5-preview-free": { aa: null, od: "meituan/longcat-2-5-preview" },
|
||||
"mimo-v2.6-pro": { aa: "mimo-v2-6-pro", od: "xiaomi/mimo-v2-6-pro" },
|
||||
"gpt-6-luna": { aa: "gpt-6-luna", od: "openai/gpt-6-luna" },
|
||||
"grok-4.7": { aa: "grok-4-7", od: "xai/grok-4-7" },
|
||||
"step-5-preview": { aa: "step-5", od: "stepfun/step-5-preview" },
|
||||
"deepseek-v4.1-flash": { aa: "deepseek-v4-1-flash", od: "deepseek/deepseek-v4-1-flash" },
|
||||
"muse-spark-1.3-contributor": { aa: "muse-spark-1-3", od: "meta/muse-spark-1-3-contributor" },
|
||||
"hy4-preview": { aa: null, od: "tencent/hy4-preview" },
|
||||
"qwen3.8-flash": { aa: "qwen3-8-flash-next", od: "alibaba/qwen3-8-flash" },
|
||||
"glm-5.3-flash": { aa: "glm-5-3-flash", od: "zhipuai/glm-5-3-flash" },
|
||||
"glm-5.3": { aa: "glm-5-3", od: "zhipuai/glm-5-3" },
|
||||
"qwen3.8-max": { aa: "qwen3-8-max", od: "alibaba/qwen3-8-max" },
|
||||
"kimi-k3": { aa: "kimi-k3", od: "moonshotai/kimi-k3" },
|
||||
"kimi-k2.7-code": { aa: "kimi-k2-7-code", od: "moonshotai/kimi-k2-7-code" },
|
||||
"qwen3.7-plus": { aa: "qwen3-7-plus", od: "alibaba/qwen3-7-plus" },
|
||||
"minimax-m3": { aa: "minimax-m3", od: "minimax/minimax-m3" },
|
||||
"deepseek-v4-pro": { aa: "deepseek-v4-pro", od: "deepseek/deepseek-v4-pro" },
|
||||
"mimo-v2.5-pro": { aa: "mimo-v2-5-pro", od: "xiaomi/mimo-v2-5-pro" },
|
||||
"mimo-v2.5": { aa: "mimo-v2-5-0424", od: "xiaomi/mimo-v2-5" }
|
||||
};
|
||||
|
||||
const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);
|
||||
|
||||
/* --- Artificial Analysis page parsing ------------------------------------ */
|
||||
function flightPayload(html) {
|
||||
const parts = [];
|
||||
const re = /self\.__next_f\.push\(\[1,"/g;
|
||||
let m;
|
||||
while ((m = re.exec(html))) {
|
||||
const start = re.lastIndex;
|
||||
const end = html.indexOf('"])</script>', start);
|
||||
if (end < 0) continue;
|
||||
try {
|
||||
parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
|
||||
} catch {
|
||||
/* ignore chunk */
|
||||
}
|
||||
}
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function extractModelObjects(html) {
|
||||
const payload = flightPayload(html);
|
||||
const out = {};
|
||||
const re = /"slug":"/g;
|
||||
let m;
|
||||
while ((m = re.exec(payload))) {
|
||||
const start = payload.lastIndexOf('{"id":"', m.index);
|
||||
if (start < 0) continue;
|
||||
let depth = 0, inStr = false, esc = false, k = start;
|
||||
for (k = start; k < payload.length; k++) {
|
||||
const c = payload[k];
|
||||
if (esc) { esc = false; continue; }
|
||||
if (c === "\\") { esc = true; continue; }
|
||||
if (c === '"') { inStr = !inStr; continue; }
|
||||
if (inStr) continue;
|
||||
if (c === "{") depth++;
|
||||
else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
|
||||
}
|
||||
let obj;
|
||||
try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
|
||||
if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function fetchAA(slug) {
|
||||
const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
|
||||
if (!res.ok) return null;
|
||||
const models = extractModelObjects(await res.text());
|
||||
return models[slug] || null;
|
||||
}
|
||||
|
||||
async function fetchOD(path) {
|
||||
try {
|
||||
const res = await fetch(`https://opencode.ai/data/${path}.json`);
|
||||
if (!res.ok) return null;
|
||||
return (await res.json()).model;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/* --- build ---------------------------------------------------------------- */
|
||||
// Names, OpenCode Go list prices and free-tier flags come from site/assets/data.js
|
||||
// (hand-maintained from the OpenCode Go docs), so the chart always shows Go pricing
|
||||
// rather than the provider's public list price.
|
||||
const CATALOG = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));
|
||||
|
||||
const rows = [];
|
||||
for (const [id, { aa, od }] of Object.entries(MODELS)) {
|
||||
const [aaModel, odModel] = await Promise.all([aa ? fetchAA(aa) : null, fetchOD(od)]);
|
||||
if (!aaModel) {
|
||||
console.warn(` ! no Artificial Analysis entry for ${id}${aa ? ` (slug ${aa})` : ""} — left empty`);
|
||||
const cat = CATALOG[id];
|
||||
rows.push({
|
||||
id, n: cat?.name || odModel?.name || id, lab: cat?.lab || odModel?.labName || "",
|
||||
free: cat ? !!cat.free : !(odModel?.priceUsdPer1M?.input),
|
||||
pi: cat?.price?.in ?? odModel?.priceUsdPer1M?.input ?? 0,
|
||||
po: cat?.price?.out ?? odModel?.priceUsdPer1M?.output ?? 0, ii: null,
|
||||
fc: null, scicode: null, tb21: null, tb40: null, tbh: null, tau2: null, taub: null, apex: null,
|
||||
analyst: null, auto: null, gpqa: null, critpt: null, hle: null, lcr: null, ifbench: null,
|
||||
mmmu: null, gdpv: null, gdppdf: null, ctx: odModel?.contextTokens ?? 1048576
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const cat = CATALOG[id];
|
||||
const price = cat?.price || odModel?.priceUsdPer1M || {};
|
||||
const ctx = odModel?.contextTokens ?? aaModel.contextWindowTokens ?? 1048576;
|
||||
const free = cat ? !!cat.free : !(odModel?.priceUsdPer1M?.input);
|
||||
rows.push({
|
||||
id,
|
||||
n: cat?.name || aaModel.name,
|
||||
lab: cat?.lab || odModel?.labName || aaModel.name,
|
||||
free,
|
||||
pi: price.in ?? price.input ?? 0,
|
||||
po: price.out ?? price.output ?? 0,
|
||||
ii: Math.round(aaModel.intelligenceIndex * 10) / 10,
|
||||
fc: pct(aaModel.frontierCode),
|
||||
scicode: pct(aaModel.scicode),
|
||||
tb21: pct(aaModel.terminalBench21),
|
||||
tb40: pct(aaModel.terminalBench40),
|
||||
tbh: pct(aaModel.terminalbenchHard),
|
||||
tau2: pct(aaModel.tau2),
|
||||
taub: pct(aaModel.tauBanking),
|
||||
apex: pct(aaModel.apexAgents),
|
||||
analyst: pct(aaModel.analystAgent),
|
||||
auto: pct(aaModel.automationBenchPartialScore),
|
||||
gpqa: pct(aaModel.gpqa),
|
||||
critpt: pct(aaModel.critpt),
|
||||
hle: pct(aaModel.hle),
|
||||
lcr: pct(aaModel.lcr),
|
||||
ifbench: pct(aaModel.ifbench),
|
||||
mmmu: pct(aaModel.mmmuPro),
|
||||
gdpv: pct(aaModel.gdpvalNormalized),
|
||||
gdppdf: pct(aaModel.gdpPdfAllPass),
|
||||
ctx
|
||||
});
|
||||
console.log(` + ${id} (index ${rows.at(-1).ii})`);
|
||||
}
|
||||
|
||||
const header = `// Evaluation data scraped from Artificial Analysis model pages (single consistent harness).
|
||||
// Values are already scaled to 0-100. Regenerate with: node scripts/refresh-evals.mjs
|
||||
export const RAW = [
|
||||
`;
|
||||
const body = rows.map((r) => " " + JSON.stringify(r)).join(",\n");
|
||||
writeFileSync(resolve(ROOT, "site/assets/evals.js"), header + body + "\n];\n");
|
||||
console.log(`\nwrote site/assets/evals.js with ${rows.length} models`);
|
||||
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env node
|
||||
// Weekly evaluation refresh.
|
||||
//
|
||||
// node scripts/sync-catalog.mjs
|
||||
//
|
||||
// Pulls three things and regenerates the auto-generated parts of the site:
|
||||
//
|
||||
// 1. OpenCode Go catalogue -> models.dev/api.json, provider "opencode-go"
|
||||
// (pricing, context window, modalities, release date, vendor description)
|
||||
// 2. Benchmark scores -> Artificial Analysis model pages
|
||||
// 3. Human-written layer -> site/assets/data.js (never touched by this script)
|
||||
//
|
||||
// Outputs
|
||||
// site/assets/catalog.js catalogue metadata for every model currently in Go
|
||||
// site/assets/evals.js benchmark scores per model
|
||||
// site/assets/snapshot.json when the data was taken + what changed
|
||||
// MODELS.md markdown rendering of the catalogue
|
||||
//
|
||||
// The human layer (site/assets/data.js) wins on anything it defines, so hand-written
|
||||
// "targeted for" descriptions survive a refresh. New models show up with the vendor
|
||||
// description and a NEEDS-WRITE-UP flag until someone writes them up.
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const UA =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";
|
||||
|
||||
/* Artificial Analysis slug overrides where the slug is not the model id. */
|
||||
const AA_OVERRIDES = {
|
||||
"gemini-4-argon-contributor": "gemini-4-argon",
|
||||
"beam-free": "reflection-beam",
|
||||
"step-5-preview": "step-5",
|
||||
"muse-spark-1.3-contributor": "muse-spark-1-3",
|
||||
"muse-spark-1.2-contributor": "muse-spark-1-2",
|
||||
"qwen3.8-flash": "qwen3-8-flash-next",
|
||||
"qwen3.7-max": "qwen3-7-max",
|
||||
"mimo-v2.5": "mimo-v2-5-0424",
|
||||
"mimo-v2.5-pro": "mimo-v2-5-pro",
|
||||
"mimo-v2.6-pro": "mimo-v2-6-pro",
|
||||
"mimo-v2.6-flash": "mimo-v2-6-flash",
|
||||
"minimax-m2.5": "minimax-m2-5",
|
||||
"minimax-m2.7": "minimax-m2-7",
|
||||
"hy3": "hy3",
|
||||
"kimi-k2.6": "kimi-k2-6",
|
||||
"glm-5.2": "glm-5-2",
|
||||
"longcat-2.0": "longcat-2-0"
|
||||
};
|
||||
|
||||
const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);
|
||||
|
||||
/* --- Artificial Analysis page parsing ------------------------------------ */
|
||||
function flightPayload(html) {
|
||||
const parts = [];
|
||||
const re = /self\.__next_f\.push\(\[1,"/g;
|
||||
let m;
|
||||
while ((m = re.exec(html))) {
|
||||
const start = re.lastIndex;
|
||||
const end = html.indexOf('"])</script>', start);
|
||||
if (end < 0) continue;
|
||||
try {
|
||||
parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
|
||||
} catch {
|
||||
/* ignore malformed chunk */
|
||||
}
|
||||
}
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function extractModelObjects(html) {
|
||||
const payload = flightPayload(html);
|
||||
const out = {};
|
||||
const re = /"slug":"/g;
|
||||
let m;
|
||||
while ((m = re.exec(payload))) {
|
||||
const start = payload.lastIndexOf('{"id":"', m.index);
|
||||
if (start < 0) continue;
|
||||
let depth = 0, inStr = false, esc = false, k = start;
|
||||
for (k = start; k < payload.length; k++) {
|
||||
const c = payload[k];
|
||||
if (esc) { esc = false; continue; }
|
||||
if (c === "\\") { esc = true; continue; }
|
||||
if (c === '"') { inStr = !inStr; continue; }
|
||||
if (inStr) continue;
|
||||
if (c === "{") depth++;
|
||||
else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
|
||||
}
|
||||
let obj;
|
||||
try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
|
||||
if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function fetchAA(slug) {
|
||||
const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
|
||||
if (!res.ok) return null;
|
||||
const models = extractModelObjects(await res.text());
|
||||
return models[slug] || null;
|
||||
}
|
||||
|
||||
function aaCandidates(id, canonical) {
|
||||
const last = (canonical || id).split("/").pop();
|
||||
const raw = [AA_OVERRIDES[id], id, last, id.replace(/-free$/, ""), id.replace(/-contributor$/, "")];
|
||||
// Artificial Analysis slugs replace dots with dashes (deepseek-v4.1-flash -> deepseek-v4-1-flash)
|
||||
const variants = [];
|
||||
for (const c of raw) {
|
||||
if (!c) continue;
|
||||
variants.push(c);
|
||||
const dashed = c.replace(/\./g, "-");
|
||||
variants.push(dashed);
|
||||
variants.push(dashed.replace(/-free$/, ""));
|
||||
}
|
||||
return [...new Set(variants)];
|
||||
}
|
||||
|
||||
/* --- sources -------------------------------------------------------------- */
|
||||
async function fetchCatalog() {
|
||||
const res = await fetch("https://models.dev/api.json", { headers: { "user-agent": UA } });
|
||||
if (!res.ok) throw new Error(`models.dev returned ${res.status}`);
|
||||
const all = await res.json();
|
||||
const go = all["opencode-go"];
|
||||
if (!go) throw new Error('no "opencode-go" provider in models.dev/api.json');
|
||||
return go.models;
|
||||
}
|
||||
|
||||
function modalitiesOf(m) {
|
||||
const inp = (m.modalities?.input || ["text"]).join(", ");
|
||||
const out = (m.modalities?.output || ["text"]).join(", ");
|
||||
return `${inp} in → ${out} out`;
|
||||
}
|
||||
|
||||
async function fetchLiveGoIds() {
|
||||
try {
|
||||
const res = await fetch("https://opencode.ai/inference/go/v1/models", { headers: { "user-agent": UA } });
|
||||
if (!res.ok) return null;
|
||||
const j = await res.json();
|
||||
return (j.data || []).map((m) => m.id);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/* --- main ------------------------------------------------------------------ */
|
||||
const catalogModels = await fetchCatalog();
|
||||
const liveIds = await fetchLiveGoIds();
|
||||
const ids = Object.keys(catalogModels);
|
||||
console.log(`catalog: ${ids.length} models in the OpenCode Go provider`);
|
||||
|
||||
const catalog = [];
|
||||
const scores = [];
|
||||
|
||||
for (const id of ids) {
|
||||
const m = catalogModels[id];
|
||||
const cost = m.cost || {};
|
||||
const free = !cost.input && !cost.output;
|
||||
const canonical = m.canonical_model_id || null;
|
||||
|
||||
let aa = null;
|
||||
for (const slug of aaCandidates(id, canonical)) {
|
||||
aa = await fetchAA(slug);
|
||||
if (aa) break;
|
||||
}
|
||||
|
||||
const row = {
|
||||
id,
|
||||
n: m.name || id,
|
||||
lab: canonical ? canonical.split("/")[0] : "",
|
||||
released: m.release_date || "",
|
||||
price: { in: cost.input ?? 0, out: cost.output ?? 0 },
|
||||
free,
|
||||
ctx: m.limit?.context ?? null,
|
||||
modalities: modalitiesOf(m),
|
||||
description: m.description || "",
|
||||
canonical,
|
||||
openWeights: !!m.open_weights
|
||||
};
|
||||
catalog.push(row);
|
||||
|
||||
scores.push({
|
||||
id,
|
||||
n: row.n,
|
||||
lab: row.lab,
|
||||
free,
|
||||
pi: row.price.in,
|
||||
po: row.price.out,
|
||||
ii: aa ? Math.round(aa.intelligenceIndex * 10) / 10 : null,
|
||||
fc: pct(aa?.frontierCode),
|
||||
scicode: pct(aa?.scicode),
|
||||
tb21: pct(aa?.terminalBench21),
|
||||
tb40: pct(aa?.terminalBench40),
|
||||
tbh: pct(aa?.terminalbenchHard),
|
||||
tau2: pct(aa?.tau2),
|
||||
taub: pct(aa?.tauBanking),
|
||||
apex: pct(aa?.apexAgents),
|
||||
analyst: pct(aa?.analystAgent),
|
||||
auto: pct(aa?.automationBenchPartialScore),
|
||||
gpqa: pct(aa?.gpqa),
|
||||
critpt: pct(aa?.critpt),
|
||||
hle: pct(aa?.hle),
|
||||
lcr: pct(aa?.lcr),
|
||||
ifbench: pct(aa?.ifbench),
|
||||
mmmu: pct(aa?.mmmuPro),
|
||||
gdpv: pct(aa?.gdpvalNormalized),
|
||||
gdppdf: pct(aa?.gdpPdfAllPass),
|
||||
ctx: row.ctx ?? 1048576
|
||||
});
|
||||
console.log(` ${aa ? "+" : "-"} ${id.padEnd(32)} ${aa ? "index " + aa.intelligenceIndex.toFixed(1) : "no public evals"}`);
|
||||
}
|
||||
|
||||
/* --- diff against the previous snapshot ---------------------------------- */
|
||||
let previous = { models: [] };
|
||||
try {
|
||||
previous = JSON.parse(
|
||||
await (await import("node:fs/promises")).readFile(resolve(ROOT, "site/assets/snapshot.json"), "utf8")
|
||||
);
|
||||
} catch {
|
||||
/* first run */
|
||||
}
|
||||
const prevIds = previous.models || [];
|
||||
const newModels = ids.filter((i) => !prevIds.includes(i));
|
||||
const removedModels = prevIds.filter((i) => !ids.includes(i));
|
||||
const prevScores = Object.fromEntries((previous.scores || []).map((s) => [s.id, s.ii]));
|
||||
const deltas = scores
|
||||
.filter((s) => s.ii !== null && prevScores[s.id] !== undefined && prevScores[s.id] !== null)
|
||||
.map((s) => ({ id: s.id, from: prevScores[s.id], to: s.ii, delta: Math.round((s.ii - prevScores[s.id]) * 10) / 10 }))
|
||||
.filter((d) => d.delta !== 0)
|
||||
.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
|
||||
|
||||
const snapshot = {
|
||||
generated: new Date().toISOString(),
|
||||
models: ids,
|
||||
scores: scores.map((s) => ({ id: s.id, ii: s.ii })),
|
||||
newModels,
|
||||
removedModels,
|
||||
deltas,
|
||||
liveCatalogSize: liveIds ? liveIds.length : null,
|
||||
notInLiveCatalog: liveIds ? ids.filter((i) => !liveIds.includes(i)) : []
|
||||
};
|
||||
|
||||
/* --- write files ---------------------------------------------------------- */
|
||||
writeFileSync(
|
||||
resolve(ROOT, "site/assets/catalog.js"),
|
||||
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
||||
// Source: models.dev/api.json (provider "opencode-go") + Artificial Analysis.
|
||||
export const CATALOG = ${JSON.stringify(catalog, null, 1)};
|
||||
|
||||
export const SNAPSHOT = ${JSON.stringify(
|
||||
{
|
||||
generated: snapshot.generated,
|
||||
models: snapshot.models.length,
|
||||
newModels: snapshot.newModels,
|
||||
removedModels: snapshot.removedModels,
|
||||
deltas: snapshot.deltas,
|
||||
notInLiveCatalog: snapshot.notInLiveCatalog
|
||||
},
|
||||
null,
|
||||
1
|
||||
)};
|
||||
`
|
||||
);
|
||||
|
||||
writeFileSync(
|
||||
resolve(ROOT, "site/assets/evals.js"),
|
||||
`// AUTO-GENERATED by scripts/sync-catalog.mjs — do not edit by hand.
|
||||
// Benchmark scores from Artificial Analysis (single consistent harness), scaled to 0-100.
|
||||
export const RAW = ${JSON.stringify(scores, null, 1)};
|
||||
`
|
||||
);
|
||||
|
||||
writeFileSync(resolve(ROOT, "site/assets/snapshot.json"), JSON.stringify(snapshot, null, 2) + "\n");
|
||||
|
||||
console.log("\nnew models:", newModels.length ? newModels.join(", ") : "none");
|
||||
console.log("removed models:", removedModels.length ? removedModels.join(", ") : "none");
|
||||
if (deltas.length) {
|
||||
console.log("score changes:");
|
||||
for (const d of deltas) console.log(` ${d.id}: ${d.from} → ${d.to} (${d.delta > 0 ? "+" : ""}${d.delta})`);
|
||||
} else {
|
||||
console.log("score changes: none");
|
||||
}
|
||||
console.log("\nwrote site/assets/catalog.js, evals.js, snapshot.json");
|
||||
Reference in New Issue
Block a user