Automate the evaluation on a weekly schedule

- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
2026-10-21 17:06:51 +00:00
parent e76281876b
commit 4e5a13f7b6
13 changed files with 2934 additions and 610 deletions
@@ -368,6 +368,158 @@ export const GO_MODELS = [
    target:
      "Repo-scale SWE work with perception in the loop: MiniMax reports one of the strongest SWE-Bench Verified results in Go, plus good browser and desktop-computer scores (BrowseComp, MCP Atlas, OSWorld). Use it when the agent has to look at a screen and then change code.",
    caveats: ["Text output only.", "Weaker than MiniMax M3.1 Flash on frontier reasoning and long-horizon tasks."]
+  },
+  {
+    id: "gpt-5.6-luna",
+    name: "GPT-5.6 Luna",
+    lab: "OpenAI",
+    released: "2026-06-01",
+    price: { in: 0.2, out: 1.2 },
+    ctx: "1.05M",
+    modalities: "text, image, PDF in → text out",
+    best: ["Fast & cheap", "High-volume coding", "Agents"],
+    blurb:
+      "Cost-efficient GPT-5.6 model for fast, high-volume workloads.",
+    target:
+      "Throughput work: many small calls, classification, extraction, routing and repetitive edits where latency and cost per call decide the outcome. Keeps the GPT tool-calling behaviour at roughly twice the price of GPT-6 Luna.",
+    caveats: ["Price steps up above 272K tokens of context."]
+  },
+  {
+    id: "muse-spark-1.2-contributor",
+    name: "Muse Spark 1.2 Contributor",
+    lab: "Meta",
+    released: "2026-08-01",
+    price: { in: 0.1, out: 0.2 },
+    ctx: "1M",
+    modalities: "text, image, video, PDF, audio in → text out",
+    best: ["Coding", "Multimodal", "Cheap"],
+    blurb:
+      "Coding-focused update to Muse Spark 1.1 with improvements in code generation, complex debugging, codebase understanding, and end-to-end developer workflows.",
+    target:
+      "The previous Muse generation — kept in Go because it is cheap and coding-focused. Pick it when you want Meta's multimodal input surface at $0.10/$0.20 and 1.3 is not required; otherwise use Muse Spark 1.3.",
+    caveats: ["Contributor tier — prompts and completions may be used to train future Meta models.", "Superseded by Muse Spark 1.3 on long-horizon agent tasks."]
+  },
+  {
+    id: "glm-5.2",
+    name: "GLM-5.2",
+    lab: "Zhipu",
+    released: "2026-06-01",
+    price: { in: 1.4, out: 4.4 },
+    ctx: "1M",
+    modalities: "text only",
+    best: ["Long-horizon coding", "Agents", "Math & science"],
+    blurb:
+      "Open flagship GLM for long-horizon coding agents and million-token context work.",
+    target:
+      "The GLM generation before 5.3: still strong on long-horizon coding agents with a million-token window, and its AIME/HMMT/IMO scores make it the maths-and-science option in the family. GLM-5.3 is the better default where budget allows.",
+    caveats: ["Text input only.", "Expensive per token ($1.40 / $4.40)."]
+  },
+  {
+    id: "hy3",
+    name: "Hy3",
+    lab: "Tencent",
+    released: "2026-04-01",
+    price: { in: 0.14, out: 0.58 },
+    ctx: "256K",
+    modalities: "text only",
+    best: ["Cheap", "Reasoning", "Instruction following", "Tool use"],
+    blurb:
+      "Tencent Hy reasoning model for coding, instruction following, and agent tasks.",
+    target:
+      "A cheap reasoning model with unusually strong instruction following (IFBench) and tool use (τ-bench) for its price. Suited to spec-driven agents and short coding loops; the 256K window is the main limitation for repo-scale work.",
+    caveats: ["256K context.", "Text input only."]
+  },
+  {
+    id: "mimo-v2.6-flash",
+    name: "MiMo-V2.6-Flash",
+    lab: "Xiaomi",
+    released: "2026-09-22",
+    price: { in: 0.14, out: 0.28 },
+    ctx: "1M",
+    modalities: "text, image, audio, video in → text out",
+    best: ["Cheap", "Multimodal", "Automation", "Coding agents"],
+    blurb:
+      "MiMo Flash model for multimodal coding agents and long-context automation.",
+    target:
+      "The cheapest multimodal agent in Go at 14¢/28¢ per 1M: bulk automation over long contexts, agents that must read images, video or audio, and the majority of mechanical coding-agent traffic.",
+    caveats: ["Text output only.", "Noticeably weaker than MiMo-V2.6-Pro on hard agentic and security tasks."]
+  },
+  {
+    id: "minimax-m2.7",
+    name: "MiniMax M2.7",
+    lab: "MiniMax",
+    released: "2026-05-01",
+    price: { in: 0.3, out: 1.2 },
+    ctx: "200K",
+    modalities: "text only",
+    best: ["Coding", "Office work", "Agentic tasks"],
+    blurb:
+      "MiniMax model for chat, coding, office work, and agentic tasks.",
+    target:
+      "The generation before MiniMax M3: general chat plus coding and office deliverables. Cheap, but the 200K window and older agentic scores make it a fallback now that M3 and M3.1 Flash exist.",
+    caveats: ["200K context — the smallest window in the Go catalogue.", "Superseded by MiniMax M3."]
+  },
+  {
+    id: "longcat-2.0",
+    name: "LongCat-2.0",
+    lab: "Meituan",
+    released: "2026-06-29",
+    price: { in: 0.3, out: 1.2 },
+    ctx: "1M",
+    modalities: "text only",
+    best: ["Coding", "Reasoning", "Long context", "Open weights"],
+    blurb:
+      "Meituan LongCat-2.0, a reasoning model with tool calling and a 1M-token context window.",
+    target:
+      "Long-context reasoning with tool calling at a modest price — useful for repo-scale analysis and document-heavy agent loops. Text-only, so perception work needs the 2.5 preview.",
+    caveats: ["Text input only.", "Superseded by LongCat 2.5 Preview, which adds image input."]
+  },
+  {
+    id: "step-5-preview-free",
+    name: "Step 5 Preview Free",
+    lab: "StepFun",
+    released: "2026-09-16",
+    price: { in: 0, out: 0 },
+    free: true,
+    ctx: "1M",
+    modalities: "text, image, video in → text out",
+    best: ["Free tier", "Coding", "Professional knowledge work", "Multimodal"],
+    blurb:
+      "Free access to StepFun's next-generation flagship base model for coding and professional knowledge work, with native text, image and video input and a 1M-token context window.",
+    target:
+      "Same model as the paid Step 5 Preview, without the token bill — useful for long agent runs and multimodal work while the paid allowance is reserved for the hard tasks.",
+    caveats: ["Limited-time free entry; not present in the live Go catalogue endpoint, so availability may vary."]
+  },
+  {
+    id: "cat-jam-free",
+    name: "cat-jam-free",
+    lab: "Undisclosed",
+    released: "2026-10-01",
+    price: { in: 0, out: 0 },
+    free: true,
+    ctx: "1M",
+    modalities: "text only",
+    best: ["Free tier", "Coding", "Agentic workflows", "Structured output"],
+    blurb:
+      "A text-only reasoning model for coding and agentic workflows, with tool calling and structured JSON outputs.",
+    target:
+      "An anonymous free tier aimed at agents: tool calling and structured JSON output make it useful as a sub-agent or a formatter that has to hand machine-readable results to the next step.",
+    caveats: ["No vendor, no model card, no public evaluation — verify behaviour before depending on it.", "Not listed in the live Go catalogue endpoint."]
+  },
+  {
+    id: "space-bunny",
+    name: "space-bunny",
+    lab: "Undisclosed",
+    released: "2026-10-01",
+    price: { in: 0.15, out: 0.6 },
+    ctx: "1M",
+    modalities: "text, image, video in → text out",
+    best: ["Coding", "Agentic tasks", "Tool use", "Multimodal"],
+    blurb:
+      "Anonymous preview reasoning model for coding, agentic tasks, tool use, and multimodal input.",
+    target:
+      "An unreleased preview model with full multimodal input and tool use, priced like the value tier. It has no vendor behind it and no public evaluation, so treat it as an experiment rather than a recommendation.",
+    caveats: ["No vendor or model card — no published evaluation.", "Not listed in the live Go catalogue endpoint."]
  }
 ];