OpenCodeGoEvals/scripts/refresh-evals.mjs

#!/usr/bin/env node
// Regenerates site/assets/evals.js from live sources.
//
//   node scripts/refresh-evals.mjs
//
// Sources
//   - Artificial Analysis model pages (https://artificialanalysis.ai/models/<slug>)
//     The benchmark payload is embedded in the page as React Server Component flight data.
//   - OpenCode Data model JSON (https://opencode.ai/data/<lab>/<model>.json)
//     for context windows and OpenCode Go list prices.
//
// Usage of the AI slug differs slightly from the OpenCode Go model id, so they are
// mapped explicitly below.

import { writeFileSync } from "node:fs";
import { GO_MODELS } from "../site/assets/data.js";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const UA =
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36";

// go model id -> { aa: artificial analysis slug, od: opencode data path }
const MODELS = {
  "minimax-m3.1-flash-free": { aa: "minimax-m3-1-flash", od: "minimax/minimax-m3-1-flash" },
  "gemini-4-argon-contributor": { aa: "gemini-4-argon", od: "google/gemini-4-argon" },
  "claude-haiku-5-5": { aa: "claude-haiku-5-5", od: "anthropic/claude-haiku-5-5" },
  "beam-free": { aa: "reflection-beam", od: "reflection/beam" },
  // No Artificial Analysis entry for these two yet — they stay empty rather than
  // borrowing a different model's numbers (LongCat 2.0 / Hy3 are not LongCat 2.5 / Hy4).
  "longcat-2.5-preview-free": { aa: null, od: "meituan/longcat-2-5-preview" },
  "mimo-v2.6-pro": { aa: "mimo-v2-6-pro", od: "xiaomi/mimo-v2-6-pro" },
  "gpt-6-luna": { aa: "gpt-6-luna", od: "openai/gpt-6-luna" },
  "grok-4.7": { aa: "grok-4-7", od: "xai/grok-4-7" },
  "step-5-preview": { aa: "step-5", od: "stepfun/step-5-preview" },
  "deepseek-v4.1-flash": { aa: "deepseek-v4-1-flash", od: "deepseek/deepseek-v4-1-flash" },
  "muse-spark-1.3-contributor": { aa: "muse-spark-1-3", od: "meta/muse-spark-1-3-contributor" },
  "hy4-preview": { aa: null, od: "tencent/hy4-preview" },
  "qwen3.8-flash": { aa: "qwen3-8-flash-next", od: "alibaba/qwen3-8-flash" },
  "glm-5.3-flash": { aa: "glm-5-3-flash", od: "zhipuai/glm-5-3-flash" },
  "glm-5.3": { aa: "glm-5-3", od: "zhipuai/glm-5-3" },
  "qwen3.8-max": { aa: "qwen3-8-max", od: "alibaba/qwen3-8-max" },
  "kimi-k3": { aa: "kimi-k3", od: "moonshotai/kimi-k3" },
  "kimi-k2.7-code": { aa: "kimi-k2-7-code", od: "moonshotai/kimi-k2-7-code" },
  "qwen3.7-plus": { aa: "qwen3-7-plus", od: "alibaba/qwen3-7-plus" },
  "minimax-m3": { aa: "minimax-m3", od: "minimax/minimax-m3" },
  "deepseek-v4-pro": { aa: "deepseek-v4-pro", od: "deepseek/deepseek-v4-pro" },
  "mimo-v2.5-pro": { aa: "mimo-v2-5-pro", od: "xiaomi/mimo-v2-5-pro" },
  "mimo-v2.5": { aa: "mimo-v2-5-0424", od: "xiaomi/mimo-v2-5" }
};

const pct = (v) => (v === null || v === undefined ? null : Math.round(v * 1000) / 10);

/* --- Artificial Analysis page parsing ------------------------------------ */
function flightPayload(html) {
  const parts = [];
  const re = /self\.__next_f\.push\(\[1,"/g;
  let m;
  while ((m = re.exec(html))) {
    const start = re.lastIndex;
    const end = html.indexOf('"])</script>', start);
    if (end < 0) continue;
    try {
      parts.push(JSON.parse('"' + html.slice(start, end).replace(/\\\//g, "/") + '"'));
    } catch {
      /* ignore chunk */
    }
  }
  return parts.join("");
}

function extractModelObjects(html) {
  const payload = flightPayload(html);
  const out = {};
  const re = /"slug":"/g;
  let m;
  while ((m = re.exec(payload))) {
    const start = payload.lastIndexOf('{"id":"', m.index);
    if (start < 0) continue;
    let depth = 0, inStr = false, esc = false, k = start;
    for (k = start; k < payload.length; k++) {
      const c = payload[k];
      if (esc) { esc = false; continue; }
      if (c === "\\") { esc = true; continue; }
      if (c === '"') { inStr = !inStr; continue; }
      if (inStr) continue;
      if (c === "{") depth++;
      else if (c === "}") { depth--; if (depth === 0) { k++; break; } }
    }
    let obj;
    try { obj = JSON.parse(payload.slice(start, k)); } catch { continue; }
    if (obj && obj.slug && obj.intelligenceIndex != null) out[obj.slug] = obj;
  }
  return out;
}

async function fetchAA(slug) {
  const res = await fetch(`https://artificialanalysis.ai/models/${slug}`, { headers: { "user-agent": UA } });
  if (!res.ok) return null;
  const models = extractModelObjects(await res.text());
  return models[slug] || null;
}

async function fetchOD(path) {
  try {
    const res = await fetch(`https://opencode.ai/data/${path}.json`);
    if (!res.ok) return null;
    return (await res.json()).model;
  } catch {
    return null;
  }
}

/* --- build ---------------------------------------------------------------- */
// Names, OpenCode Go list prices and free-tier flags come from site/assets/data.js
// (hand-maintained from the OpenCode Go docs), so the chart always shows Go pricing
// rather than the provider's public list price.
const CATALOG = Object.fromEntries(GO_MODELS.map((m) => [m.id, m]));

const rows = [];
for (const [id, { aa, od }] of Object.entries(MODELS)) {
  const [aaModel, odModel] = await Promise.all([aa ? fetchAA(aa) : null, fetchOD(od)]);
  if (!aaModel) {
    console.warn(`  ! no Artificial Analysis entry for ${id}${aa ? ` (slug ${aa})` : ""} — left empty`);
    const cat = CATALOG[id];
    rows.push({
      id, n: cat?.name || odModel?.name || id, lab: cat?.lab || odModel?.labName || "",
      free: cat ? !!cat.free : !(odModel?.priceUsdPer1M?.input),
      pi: cat?.price?.in ?? odModel?.priceUsdPer1M?.input ?? 0,
      po: cat?.price?.out ?? odModel?.priceUsdPer1M?.output ?? 0, ii: null,
      fc: null, scicode: null, tb21: null, tb40: null, tbh: null, tau2: null, taub: null, apex: null,
      analyst: null, auto: null, gpqa: null, critpt: null, hle: null, lcr: null, ifbench: null,
      mmmu: null, gdpv: null, gdppdf: null, ctx: odModel?.contextTokens ?? 1048576
    });
    continue;
  }
  const cat = CATALOG[id];
  const price = cat?.price || odModel?.priceUsdPer1M || {};
  const ctx = odModel?.contextTokens ?? aaModel.contextWindowTokens ?? 1048576;
  const free = cat ? !!cat.free : !(odModel?.priceUsdPer1M?.input);
  rows.push({
    id,
    n: cat?.name || aaModel.name,
    lab: cat?.lab || odModel?.labName || aaModel.name,
    free,
    pi: price.in ?? price.input ?? 0,
    po: price.out ?? price.output ?? 0,
    ii: Math.round(aaModel.intelligenceIndex * 10) / 10,
    fc: pct(aaModel.frontierCode),
    scicode: pct(aaModel.scicode),
    tb21: pct(aaModel.terminalBench21),
    tb40: pct(aaModel.terminalBench40),
    tbh: pct(aaModel.terminalbenchHard),
    tau2: pct(aaModel.tau2),
    taub: pct(aaModel.tauBanking),
    apex: pct(aaModel.apexAgents),
    analyst: pct(aaModel.analystAgent),
    auto: pct(aaModel.automationBenchPartialScore),
    gpqa: pct(aaModel.gpqa),
    critpt: pct(aaModel.critpt),
    hle: pct(aaModel.hle),
    lcr: pct(aaModel.lcr),
    ifbench: pct(aaModel.ifbench),
    mmmu: pct(aaModel.mmmuPro),
    gdpv: pct(aaModel.gdpvalNormalized),
    gdppdf: pct(aaModel.gdpPdfAllPass),
    ctx
  });
  console.log(`  + ${id} (index ${rows.at(-1).ii})`);
}

const header = `// Evaluation data scraped from Artificial Analysis model pages (single consistent harness).
// Values are already scaled to 0-100. Regenerate with: node scripts/refresh-evals.mjs
export const RAW = [
`;
const body = rows.map((r) => " " + JSON.stringify(r)).join(",\n");
writeFileSync(resolve(ROOT, "site/assets/evals.js"), header + body + "\n];\n");
console.log(`\nwrote site/assets/evals.js with ${rows.length} models`);