feat(admin): AI model tier selection — internal repairs on Opus 5, admin-switchable #5440
5 changed files+243−9
Addedsrc/__tests__/ai-model-tiers.test.ts+44−0View fileUnifiedSplit
@@ -0,0 +1,44 @@
1import { afterEach, describe, expect, it } from "bun:test";
2import {
3 __resetModelOverridesForTests,
4 AVAILABLE_MODELS,
5 internalModel,
6 MODEL_HAIKU,
7 MODEL_OPUS,
8 MODEL_SONNET,
9 modelForTask,
10} from "../lib/ai-client";
11
12afterEach(() => {
13 __resetModelOverridesForTests();
14 delete process.env.AI_FORCE_SONNET;
15});
16
17describe("model tiers", () => {
18 it("internal tier defaults to Opus 5 — the owner's choice for self-repair", () => {
19 expect(internalModel()).toBe(MODEL_OPUS);
20 expect(MODEL_OPUS).toBe("claude-opus-5");
21 });
22
23 it("light tasks stay on Haiku, code tasks on Sonnet, unknown tasks never downgrade", () => {
24 expect(modelForTask("commit-message")).toBe(MODEL_HAIKU);
25 expect(modelForTask("code-review")).toBe(MODEL_SONNET);
26 expect(modelForTask("definitely-not-a-task" as never)).toBe(MODEL_SONNET);
27 });
28
29 it("AI_FORCE_SONNET kill-switch still routes everything to the standard tier", () => {
30 process.env.AI_FORCE_SONNET = "1";
31 expect(modelForTask("commit-message")).toBe(MODEL_SONNET);
32 });
33
34 it("the admin catalog contains only current model aliases", () => {
35 const ids = AVAILABLE_MODELS.map((m) => m.id);
36 expect(ids).toContain("claude-opus-5");
37 expect(ids).toContain("claude-sonnet-5");
38 expect(ids).toContain("claude-haiku-4-5");
39 // No dated snapshot IDs in the selector — aliases only.
40 for (const id of ids) {
41 expect(id).not.toMatch(/\d{8}$/);
42 }
43 });
44});
Modifiedsrc/lib/ai-client.ts+106−5View fileUnifiedSplit
@@ -37,6 +37,102 @@ export function __resetAnthropicClientForTests(): void {
3737export const MODEL_SONNET = "claude-sonnet-4-6";
3838/** Light-task model — never reference directly; route through modelForTask() */
3939export const MODEL_HAIKU = "claude-haiku-4-5-20251001";
40/** Internal-operations model — platform self-repairs, incident RCA. */
41export const MODEL_OPUS = "claude-opus-5";
42
43/**
44 * Model catalog for the admin selector (/admin — AI models section).
45 * IDs are current Anthropic aliases; pricing shown per 1M tokens in/out.
46 */
47export const AVAILABLE_MODELS: ReadonlyArray<{
48 id: string;
49 label: string;
50 pricing: string;
51}> = [
52 { id: "claude-opus-5", label: "Claude Opus 5", pricing: "$5 / $25 per MTok" },
53 { id: "claude-opus-4-8", label: "Claude Opus 4.8", pricing: "$5 / $25 per MTok" },
54 { id: "claude-sonnet-5", label: "Claude Sonnet 5", pricing: "$3 / $15 per MTok" },
55 { id: "claude-sonnet-4-6", label: "Claude Sonnet 4.6", pricing: "$3 / $15 per MTok" },
56 { id: "claude-haiku-4-5", label: "Claude Haiku 4.5", pricing: "$1 / $5 per MTok" },
57];
58
59/**
60 * Admin-selectable model tiers, stored in system_flags so the owner can
61 * change them from /admin without a deploy:
62 *
63 * ai_model_internal — platform-maintenance AI (incident RCA, self-repair).
64 * Runs on the OWNER's Anthropic API account; default
65 * Opus 5 per owner decision 2026-08-08.
66 * ai_model_standard — everything that writes/judges code for users.
67 * ai_model_light — the Haiku allowlist below.
68 *
69 * modelForTask() must stay synchronous (52 call sites), so flags are read
70 * through a 60s in-memory cache refreshed fire-and-forget. Until the first
71 * refresh lands the compiled-in defaults apply — a DB blip can never break
72 * an AI call, it just means defaults.
73 */
74export const AI_MODEL_FLAG_KEYS = {
75 internal: "ai_model_internal",
76 standard: "ai_model_standard",
77 light: "ai_model_light",
78} as const;
79
80type ModelOverrides = { internal?: string; standard?: string; light?: string };
81let _overrides: ModelOverrides = {};
82let _overridesFetchedAt = 0;
83let _overridesRefreshing = false;
84const OVERRIDES_TTL_MS = 60_000;
85
86const KNOWN_MODEL_IDS = new Set(AVAILABLE_MODELS.map((m) => m.id));
87
88function maybeRefreshOverrides(): void {
89 const now = Date.now();
90 if (_overridesRefreshing || now - _overridesFetchedAt < OVERRIDES_TTL_MS) {
91 return;
92 }
93 _overridesRefreshing = true;
94 // Dynamic import avoids a static cycle (admin.ts → db → … → ai callers).
95 void import("./admin")
96 .then(async ({ getFlag }) => {
97 const [internal, standard, light] = await Promise.all([
98 getFlag(AI_MODEL_FLAG_KEYS.internal),
99 getFlag(AI_MODEL_FLAG_KEYS.standard),
100 getFlag(AI_MODEL_FLAG_KEYS.light),
101 ]);
102 // Only accept catalog models — a typo'd flag must never send every
103 // AI call to a 404ing model id.
104 _overrides = {
105 internal: internal && KNOWN_MODEL_IDS.has(internal) ? internal : undefined,
106 standard: standard && KNOWN_MODEL_IDS.has(standard) ? standard : undefined,
107 light: light && KNOWN_MODEL_IDS.has(light) ? light : undefined,
108 };
109 _overridesFetchedAt = Date.now();
110 })
111 .catch(() => {
112 _overridesFetchedAt = Date.now(); // don't hot-loop on failure
113 })
114 .finally(() => {
115 _overridesRefreshing = false;
116 });
117}
118
119/** Test-only: reset the flag cache. */
120export function __resetModelOverridesForTests(): void {
121 _overrides = {};
122 _overridesFetchedAt = 0;
123 _overridesRefreshing = false;
124}
125
126/**
127 * Model for INTERNAL platform operations — the AI that maintains gluecron
128 * itself (deploy-failure RCA, self-repair). Distinct from customer-facing
129 * tiers so the owner can run repairs on Opus while customer features run
130 * on cost-appropriate models billed to their usage.
131 */
132export function internalModel(): string {
133 maybeRefreshOverrides();
134 return _overrides.internal ?? MODEL_OPUS;
135}
40136
41137/**
42138 * Task → model routing.
@@ -79,14 +175,19 @@ const HAIKU_ALLOWLIST: ReadonlySet<AiTask> = new Set<AiTask>([
79175]);
80176
81177/**
82 * Resolve the model for a task. Allowlisted light tasks get Haiku; everything
83 * else (including unknown task strings) gets Sonnet. `AI_FORCE_SONNET=1`
84 * forces Sonnet for all tasks.
178 * Resolve the model for a task. Allowlisted light tasks get the light-tier
179 * model; everything else (including unknown task strings) gets the standard
180 * tier. Admin flag overrides (see AI_MODEL_FLAG_KEYS) apply per tier;
181 * `AI_FORCE_SONNET=1` forces the standard tier for all tasks.
85182 */
86183export function modelForTask(task: AiTask): string {
184 maybeRefreshOverrides();
185 const standard = _overrides.standard ?? MODEL_SONNET;
87186 // Read at call time so the kill-switch works without a restart.
88 if (process.env.AI_FORCE_SONNET === "1") return MODEL_SONNET;
89 return HAIKU_ALLOWLIST.has(task) ? MODEL_HAIKU : MODEL_SONNET;
187 if (process.env.AI_FORCE_SONNET === "1") return standard;
188 return HAIKU_ALLOWLIST.has(task)
189 ? (_overrides.light ?? MODEL_HAIKU)
190 : standard;
90191}
91192
92193/**
Modifiedsrc/lib/ai-cost-tracker.ts+9−2View fileUnifiedSplit
@@ -51,15 +51,22 @@ export const MODEL_PRICING: Record<string, ModelPrice> = {
5151 "claude-sonnet-4-5": { inputCentsPer1k: 0.3, outputCentsPer1k: 1.5 },
5252 "claude-sonnet-4-6": { inputCentsPer1k: 0.3, outputCentsPer1k: 1.5 },
5353 "claude-sonnet-4-7": { inputCentsPer1k: 0.3, outputCentsPer1k: 1.5 },
54 // Sonnet 5 — $3 / $15 per 1M list (intro $2/$10 through 2026-08-31;
55 // we record list rates so aggregates don't dip then jump).
56 "claude-sonnet-5": { inputCentsPer1k: 0.3, outputCentsPer1k: 1.5 },
5457 // Haiku 4.5 — $1 / $5 per 1M tokens.
5558 "claude-haiku-4-5-20251001": {
5659 inputCentsPer1k: 0.1,
5760 outputCentsPer1k: 0.5,
5861 },
5962 "claude-haiku-4-5": { inputCentsPer1k: 0.1, outputCentsPer1k: 0.5 },
60 // Opus 4 family — $15 / $75 per 1M tokens.
63 // Opus 4 legacy — $15 / $75 per 1M tokens.
6164 "claude-opus-4": { inputCentsPer1k: 1.5, outputCentsPer1k: 7.5 },
62 "claude-opus-4-7": { inputCentsPer1k: 1.5, outputCentsPer1k: 7.5 },
65 // Opus 4.6+ and Opus 5 — $5 / $25 per 1M tokens.
66 "claude-opus-4-6": { inputCentsPer1k: 0.5, outputCentsPer1k: 2.5 },
67 "claude-opus-4-7": { inputCentsPer1k: 0.5, outputCentsPer1k: 2.5 },
68 "claude-opus-4-8": { inputCentsPer1k: 0.5, outputCentsPer1k: 2.5 },
69 "claude-opus-5": { inputCentsPer1k: 0.5, outputCentsPer1k: 2.5 },
6370};
6471
6572/** Fallback pricing if we get a model id we don't recognise. Conservative
Modifiedsrc/lib/ai-incident.ts+5−2View fileUnifiedSplit
@@ -25,9 +25,9 @@ import {
2525} from "../db/schema";
2626import { getDefaultBranch, listCommits } from "../git/repository";
2727import {
28 MODEL_SONNET,
2928 extractText,
3029 getAnthropic,
30 internalModel,
3131 isAiAvailable,
3232 parseJsonResponse,
3333} from "./ai-client";
@@ -123,7 +123,10 @@ async function askClaudeForAnalysis(
123123 try {
124124 const client = getAnthropic();
125125 const message = await client.messages.create({
126 model: MODEL_SONNET,
126 // Internal-tier model (admin-selectable at /admin, default Opus 5):
127 // this is the platform repairing ITSELF, billed to the owner's
128 // Anthropic account — not a customer-usage call.
129 model: internalModel(),
127130 max_tokens: 1024,
128131 messages: [
129132 {
Modifiedsrc/routes/admin.tsx+79−0View fileUnifiedSplit
@@ -46,6 +46,11 @@ import {
4646 revokeSiteAdmin,
4747 setFlag,
4848} from "../lib/admin";
49import {
50 AI_MODEL_FLAG_KEYS,
51 AVAILABLE_MODELS,
52 MODEL_SONNET,
53} from "../lib/ai-client";
4954import { audit } from "../lib/notify";
5055import { sendDigestsToAll, sendDigestForUser } from "../lib/email-digest";
5156import {
@@ -4088,6 +4093,62 @@ admin.get("/admin/flags", async (c) => {
40884093 </div>
40894094 </section>
40904095
4096 <form method="post" action="/admin/flags/ai-models" class="adm-flags-card" style="margin-bottom: 24px">
4097 <div class="adm-flags-card-body">
4098 <div style="font-weight: 600; font-size: 15px; margin-bottom: 4px">
4099 AI models
4100 </div>
4101 <div class="adm-flags-hint" style="margin-bottom: 12px">
4102 Which Claude model each tier runs on. <strong>Internal</strong> is
4103 the platform maintaining itself (incident RCA, self-repair) on the
4104 owner's Anthropic API account. <strong>Standard</strong> and{" "}
4105 <strong>Light</strong> serve user-facing AI features, metered per
4106 user on <a href="/billing/usage">/billing/usage</a>. Changes apply
4107 within 60 seconds, no deploy.
4108 </div>
4109 {(
4110 [
4111 ["internal", AI_MODEL_FLAG_KEYS.internal, "claude-opus-5", "Internal operations (platform self-repair)"],
4112 ["standard", AI_MODEL_FLAG_KEYS.standard, MODEL_SONNET, "Standard (code review, generation, chat)"],
4113 ["light", AI_MODEL_FLAG_KEYS.light, "claude-haiku-4-5", "Light (commit messages, triage, labels)"],
4114 ] as const
4115 ).map(([tier, flagKey, def, label]) => {
4116 const current = existingMap.get(flagKey) ?? "";
4117 return (
4118 <div class="adm-flags-field">
4119 <div class="adm-flags-field-head">
4120 <label for={`flag-${flagKey}`} class="adm-flags-key">{label}</label>
4121 {current && <span class="adm-flags-mono">overridden</span>}
4122 </div>
4123 <select
4124 id={`flag-${flagKey}`}
4125 name={flagKey}
4126 class="adm-flags-input"
4127 aria-label={label}
4128 >
4129 <option value="" selected={!current}>
4130 Built-in default ({def})
4131 </option>
4132 {AVAILABLE_MODELS.map((m) => (
4133 <option value={m.id} selected={current === m.id}>
4134 {m.label} — {m.pricing}
4135 </option>
4136 ))}
4137 </select>
4138 </div>
4139 );
4140 })}
4141 </div>
4142 <div class="adm-flags-card-foot">
4143 <span class="adm-flags-foot-hint">
4144 Only catalog models are accepted — a stale value falls back to the default.
4145 </span>
4146 <button type="submit" class="adm-flags-btn adm-flags-btn-primary">
4147 Save AI models
4148 </button>
4149 </div>
4150 </form>
4151
40914152 <form method="post" action="/admin/flags" class="adm-flags-card">
40924153 <div class="adm-flags-card-body">
40934154 {keys.map((k) => {
@@ -4133,6 +4194,24 @@ admin.get("/admin/flags", async (c) => {
41334194 );
41344195});
41354196
4197admin.post("/admin/flags/ai-models", async (c) => {
4198 const g = await gate(c);
4199 if (g instanceof Response) return g;
4200 const { user } = g;
4201 const body = await c.req.parseBody();
4202 const validIds = new Set(AVAILABLE_MODELS.map((m) => m.id));
4203 for (const flagKey of Object.values(AI_MODEL_FLAG_KEYS)) {
4204 const v = String(body[flagKey] ?? "");
4205 // "" clears the override (built-in default); anything else must be a
4206 // catalog model so a form glitch can't route AI calls to a 404 id.
4207 if (v === "" || validIds.has(v)) {
4208 await setFlag(flagKey, v, user.id);
4209 }
4210 }
4211 await audit({ userId: user.id, action: "admin.flags.ai_models.save" });
4212 return c.redirect("/admin/flags");
4213});
4214
41364215admin.post("/admin/flags", async (c) => {
41374216 const g = await gate(c);
41384217 if (g instanceof Response) return g;
41394218
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts