devadmin
4e5a13f7b6
Deploy to Cloudflare Pages / deploy (push) Successful in 30s
- scripts/sync-catalog.mjs: rebuilds the Go catalogue from models.dev and benchmark scores from Artificial Analysis, and records what changed in snapshot.json - scripts/build-models-md.mjs: regenerates MODELS.md from the data layers - scripts/notify-new-models.mjs: files an issue when a new model needs a write-up - weekly-eval.yml: cron every Monday 05:23 UTC -> refresh -> commit -> Pages deploy - site now renders every model in the catalogue (33) with a data-snapshot stamp; auto-added models are flagged 'no write-up' until described - replaced scripts/refresh-evals.mjs with the wider sync script
81 lines
2.8 KiB
JavaScript
81 lines
2.8 KiB
JavaScript
#!/usr/bin/env node
|
|
// Opens a Gitea issue when the weekly refresh finds models that need a write-up.
|
|
//
|
|
// node scripts/notify-new-models.mjs
|
|
//
|
|
// Reads site/assets/snapshot.json (written by sync-catalog.mjs) and, if the catalogue
|
|
// gained models, files an issue listing them so somebody writes the "targeted for"
|
|
// descriptions in site/assets/data.js.
|
|
//
|
|
// Uses GITHUB_API_URL / GITEA_TOKEN (or GITHUB_TOKEN) from the environment.
|
|
|
|
import { readFileSync } from "node:fs";
|
|
import { dirname, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
const snapshot = JSON.parse(readFileSync(resolve(ROOT, "site/assets/snapshot.json"), "utf8"));
|
|
|
|
const api = process.env.GITHUB_API_URL || "https://git.wallacenet.ca/api/v1";
|
|
const repo = process.env.GITHUB_REPOSITORY || "devadmin/OpenCodeGoEvals";
|
|
const token = process.env.GITEA_TOKEN || process.env.GITHUB_TOKEN;
|
|
|
|
if (!snapshot.newModels?.length) {
|
|
console.log("no new models — nothing to report");
|
|
if (snapshot.deltas?.length) {
|
|
console.log("score changes since last refresh:");
|
|
for (const d of snapshot.deltas) console.log(` ${d.id}: ${d.from} → ${d.to}`);
|
|
}
|
|
process.exit(0);
|
|
}
|
|
|
|
const { CATALOG } = await import(resolve(ROOT, "site/assets/catalog.js"));
|
|
const rows = snapshot.newModels
|
|
.map((id) => CATALOG.find((c) => c.id === id))
|
|
.filter(Boolean)
|
|
.map(
|
|
(c) =>
|
|
`| ${c.name} | ${c.lab || "undisclosed"} | $${c.price.in}/${c.price.out} | ${c.ctx ? (c.ctx / 1e6).toFixed(0) + "M" : "n/a"} | ${c.description || "—"} |`
|
|
);
|
|
|
|
const body = [
|
|
"The weekly catalogue refresh found model(s) that are live in OpenCode Go but have no",
|
|
"hand-written write-up yet.",
|
|
"",
|
|
"| Model | Lab | Price $/1M | Context | Vendor description |",
|
|
"|---|---|---|---|---|",
|
|
...rows,
|
|
"",
|
|
"To add them:",
|
|
"",
|
|
"1. Add an entry to `site/assets/data.js` under `GO_MODELS` with `id`, `best` (targeting tags),",
|
|
" `blurb`, `target` and `caveats`.",
|
|
"2. Run `node scripts/build-models-md.mjs` to regenerate `MODELS.md`.",
|
|
"3. Commit — CI publishes the site.",
|
|
"",
|
|
`Detected ${new Date().toISOString().slice(0, 10)}.`
|
|
].join("\n");
|
|
|
|
if (!token) {
|
|
console.log("no token in environment — would file an issue with:\n");
|
|
console.log(body);
|
|
process.exit(0);
|
|
}
|
|
|
|
const res = await fetch(`${api}/repos/${repo}/issues`, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
Authorization: `token ${token}`,
|
|
"User-Agent": "opencode-go-evals-bot"
|
|
},
|
|
body: JSON.stringify({ title: `New model(s) need write-ups: ${snapshot.newModels.join(", ")}`, body })
|
|
});
|
|
|
|
if (!res.ok) {
|
|
console.error(`could not create issue: ${res.status} ${await res.text()}`);
|
|
process.exit(1);
|
|
}
|
|
const issue = await res.json();
|
|
console.log(`opened issue #${issue.number}: ${issue.html_url}`);
|