feat(admin): show whether the AI key actually works, not just what it spent #5596
3 changed files+435−0
Addedsrc/__tests__/ai-provider-health.test.ts+152−0View fileUnifiedSplit
@@ -0,0 +1,152 @@
1/**
2 * The panel that would have caught a four-week AI outage on day one.
3 *
4 * The key stopped working on 2026-08-04. Every AI feature deferred quietly for
5 * the rest of the month, and the only way anyone found out was an agent SSHing
6 * into the box and curling the API by hand. Meanwhile /admin/ai-costs rendered
7 * a perfectly tidy $1.75 month — a page about AI that could not tell you the
8 * AI was dead.
9 *
10 * These tests are mostly about WORDING, which is unusual for a test file and
11 * is the point: "AI unavailable" and "your key's account is out of credit"
12 * are the same fact and different products. The second is the only one the
13 * person who can fix it can act on.
14 */
15
16import { describe, expect, test } from "bun:test";
17import {
18 adviseOnProviderError,
19 describeLastSuccess,
20 probeProvider,
21} from "../lib/ai-provider-health";
22
23describe("the advice names the actual remedy", () => {
24 test("out of credit is not confused with a bad key", () => {
25 const credit = adviseOnProviderError(400, "Your credit balance is too low to access the Anthropic API");
26 expect(credit).toMatch(/out of credit/i);
27 // The trap this platform actually hit: funds added to the wrong org.
28 expect(credit).toMatch(/organisation|account/i);
29 expect(credit).not.toMatch(/rotate/i);
30 });
31
32 test("a rejected key says rotate, and says it is NOT a balance problem", () => {
33 const bad = adviseOnProviderError(401, "invalid x-api-key");
34 expect(bad).toMatch(/rotate/i);
35 expect(bad).toMatch(/not merely out of credit/i);
36 });
37
38 test("a provider 500 says plainly that it is not our fault", () => {
39 const up = adviseOnProviderError(503, "overloaded");
40 expect(up).toMatch(/provider is failing, not us/i);
41 // The reassurance that matters during an AI outage on a git platform.
42 expect(up).toMatch(/git, CI, merges and deploys/i);
43 });
44
45 test("no answer at all points at the network, not the key", () => {
46 expect(adviseOnProviderError(null, "no response within 20s")).toMatch(/outbound network/i);
47 });
48
49 test("an unrecognised error gets NO advice rather than a guess", () => {
50 // A confident wrong instruction costs more than none: the reader follows
51 // it. Silence hands them the provider's own words instead.
52 expect(adviseOnProviderError(418, "something we have never seen")).toBe("");
53 });
54});
55
56describe("the ledger tells the truth when nobody presses the button", () => {
57 const now = new Date("2026-08-31T00:00:00Z");
58
59 test("never billed is bad, not blank", () => {
60 // A page showing nothing looks exactly like a page showing health.
61 const d = describeLastSuccess(null, now);
62 expect(d.tone).toBe("bad");
63 expect(d.text).toMatch(/nothing has proven the key works/i);
64 });
65
66 test("a four-week gap is called out as a probable dead key", () => {
67 const d = describeLastSuccess(new Date("2026-08-04T00:00:00Z"), now);
68 expect(d.tone).toBe("bad");
69 expect(d.text).toContain("27 days ago");
70 expect(d.text).toMatch(/key stopped working/i);
71 });
72
73 test("hours of quiet is not an alarm", () => {
74 const d = describeLastSuccess(new Date("2026-08-30T12:00:00Z"), now);
75 expect(d.tone).toBe("ok");
76 });
77
78 test("a few days is warned about without being called broken", () => {
79 const d = describeLastSuccess(new Date("2026-08-27T00:00:00Z"), now);
80 expect(d.tone).toBe("warn");
81 expect(d.text).toMatch(/not yet unusual/i);
82 });
83});
84
85describe("the probe reports what happened, not an interpretation of it", () => {
86 test("a missing key is answered without a network call", async () => {
87 let called = false;
88 const p = await probeProvider({
89 apiKey: "",
90 fetchImpl: (async () => {
91 called = true;
92 return new Response("{}");
93 }) as unknown as typeof fetch,
94 });
95 expect(called).toBe(false);
96 expect(p.ok).toBe(false);
97 expect(p.detail).toMatch(/not set/i);
98 });
99
100 test("the provider's own sentence survives to the screen", async () => {
101 const p = await probeProvider({
102 apiKey: "k",
103 fetchImpl: (async () =>
104 new Response(
105 JSON.stringify({
106 type: "error",
107 error: { type: "invalid_request_error", message: "Your credit balance is too low" },
108 }),
109 { status: 400 }
110 )) as unknown as typeof fetch,
111 });
112 expect(p.ok).toBe(false);
113 expect(p.status).toBe(400);
114 // Verbatim — not "AI unavailable".
115 expect(p.detail).toBe("Your credit balance is too low");
116 expect(p.advice).toMatch(/out of credit/i);
117 });
118
119 test("a non-JSON body is shown rather than swallowed", async () => {
120 const p = await probeProvider({
121 apiKey: "k",
122 fetchImpl: (async () =>
123 new Response("<html>502 Bad Gateway</html>", { status: 502 })) as unknown as typeof fetch,
124 });
125 expect(p.detail).toContain("502 Bad Gateway");
126 });
127
128 test("a thrown fetch is an outcome, never an exception", async () => {
129 // This runs inside an admin page render. A health check that can crash
130 // the page it reports on turns a provider outage into what looks like a
131 // platform bug.
132 const p = await probeProvider({
133 apiKey: "k",
134 fetchImpl: (async () => {
135 throw new Error("getaddrinfo ENOTFOUND api.anthropic.com");
136 }) as unknown as typeof fetch,
137 });
138 expect(p.ok).toBe(false);
139 expect(p.status).toBeNull();
140 expect(p.detail).toContain("ENOTFOUND");
141 });
142
143 test("success says so and carries no error text", async () => {
144 const p = await probeProvider({
145 apiKey: "k",
146 fetchImpl: (async () => new Response("{}", { status: 200 })) as unknown as typeof fetch,
147 });
148 expect(p.ok).toBe(true);
149 expect(p.detail).toBe("");
150 expect(p.advice).toBe("");
151 });
152});
Addedsrc/lib/ai-provider-health.ts+189−0View fileUnifiedSplit
@@ -0,0 +1,189 @@
1/**
2 * Can we actually reach the model right now, and if not, what did the
3 * provider say?
4 *
5 * WHY THIS EXISTS. Since 2026-08-04 this platform's Anthropic key has been
6 * returning `400 credit balance is too low`. Every AI feature has been
7 * silently deferring for weeks. The owner could not see that anywhere: the
8 * only way anyone learned it was an agent SSHing into the box and curling the
9 * API by hand, then reporting back. That is a diagnosis the product should be
10 * able to perform on itself — the whole complaint behind "when the owner
11 * can't find something in admin, build it into the product, not a shell
12 * script".
13 *
14 * WHAT MAKES THIS HONEST. Three separate facts, never conflated:
15 *
16 * 1. Whether a call SUCCEEDS right now (a live probe, on demand).
17 * 2. What the provider SAID when it last failed (verbatim, truncated —
18 * "credit balance is too low" and "invalid x-api-key" call for opposite
19 * actions, and a humanised "AI unavailable" hides which one it was).
20 * 3. When a real, billed call last succeeded (from the cost ledger).
21 *
22 * The third is the one that catches the failure mode this file exists for.
23 * A probe that has never been clicked reports nothing, and a page that shows
24 * nothing looks exactly like a page that shows health — the "unrun check
25 * indistinguishable from a passing one" trap. The ledger's last row is
26 * evidence that exists whether or not anyone pressed anything.
27 *
28 * COST. The probe is one Haiku call with max_tokens 1: roughly five
29 * millionths of a dollar. It runs ONLY when explicitly requested, never on
30 * page load, because a health check that bills on every render is a bill
31 * nobody agreed to.
32 */
33
34const PROBE_MODEL = "claude-haiku-4-5-20251001";
35const PROBE_TIMEOUT_MS = 20_000;
36
37export interface ProviderProbe {
38 ok: boolean;
39 /** HTTP status, or null when the request never got an answer. */
40 status: number | null;
41 /** The provider's own error text, truncated. Verbatim on purpose. */
42 detail: string;
43 /** What to DO about it, when the error is one we recognise. */
44 advice: string;
45 checkedAt: Date;
46}
47
48/**
49 * Map a provider failure to the action it calls for.
50 *
51 * Deliberately narrow: only errors whose remedy is unambiguous get advice.
52 * Everything else gets the provider's own words and no guess, because a
53 * confident wrong instruction costs more than none — the reader would follow
54 * it.
55 */
56export function adviseOnProviderError(status: number | null, detail: string): string {
57 const d = detail.toLowerCase();
58 if (status === null) {
59 return "The request never reached the provider. Check the container's outbound network before touching the key.";
60 }
61 if (d.includes("credit balance")) {
62 return "The key's ACCOUNT is out of credit. Adding funds to a different account under the same login will not fix it — check which organisation this key belongs to.";
63 }
64 if (status === 401 || d.includes("invalid x-api-key") || d.includes("authentication")) {
65 return "The key is rejected, not merely out of credit. Rotate ANTHROPIC_API_KEY in /opt/gluecron/.env and redeploy.";
66 }
67 if (status === 429) {
68 return "Rate limited. Nothing is broken; retry shortly.";
69 }
70 if (status === 404 && d.includes("model")) {
71 return `The probe model (${PROBE_MODEL}) was not found for this key. The key may lack access to it.`;
72 }
73 if (status >= 500) {
74 return "The provider is failing, not us. Nothing to fix here; git, CI, merges and deploys make no model calls and are unaffected.";
75 }
76 return "";
77}
78
79/**
80 * One live call against the configured key.
81 *
82 * Never throws. A health check that can crash the page it reports on is worse
83 * than no health check, because the failure it produces looks like a platform
84 * bug rather than the provider outage it was trying to describe.
85 */
86export async function probeProvider(
87 opts: { apiKey?: string; fetchImpl?: typeof fetch } = {}
88): Promise<ProviderProbe> {
89 const checkedAt = new Date();
90 const apiKey = opts.apiKey ?? process.env["ANTHROPIC_API_KEY"] ?? "";
91 if (!apiKey) {
92 return {
93 ok: false,
94 status: null,
95 detail: "ANTHROPIC_API_KEY is not set in this container's environment",
96 advice:
97 "Set ANTHROPIC_API_KEY in /opt/gluecron/.env and redeploy. Until then every AI feature defers; nothing else is affected.",
98 checkedAt,
99 };
100 }
101
102 const doFetch = opts.fetchImpl ?? fetch;
103 const controller = new AbortController();
104 const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
105 try {
106 const res = await doFetch("https://api.anthropic.com/v1/messages", {
107 method: "POST",
108 headers: {
109 "x-api-key": apiKey,
110 "anthropic-version": "2023-06-01",
111 "content-type": "application/json",
112 },
113 body: JSON.stringify({
114 model: PROBE_MODEL,
115 max_tokens: 1,
116 messages: [{ role: "user", content: "." }],
117 }),
118 signal: controller.signal,
119 });
120 if (res.ok) {
121 return { ok: true, status: res.status, detail: "", advice: "", checkedAt };
122 }
123 const raw = (await res.text().catch(() => "")).slice(0, 600);
124 // The API nests the useful sentence; surface it, and fall back to the
125 // whole body rather than to a shrug.
126 let detail = raw;
127 try {
128 const parsed = JSON.parse(raw) as { error?: { message?: string } };
129 if (parsed?.error?.message) detail = parsed.error.message;
130 } catch {
131 /* not JSON — the raw body is still the most honest thing to show */
132 }
133 return {
134 ok: false,
135 status: res.status,
136 detail,
137 advice: adviseOnProviderError(res.status, detail),
138 checkedAt,
139 };
140 } catch (err) {
141 const msg = (err as Error)?.name === "AbortError"
142 ? `no response within ${PROBE_TIMEOUT_MS / 1000}s`
143 : (err as Error).message;
144 return {
145 ok: false,
146 status: null,
147 detail: msg,
148 advice: adviseOnProviderError(null, msg),
149 checkedAt,
150 };
151 } finally {
152 clearTimeout(timer);
153 }
154}
155
156/**
157 * How to describe the gap since the last billed call.
158 *
159 * Separated from rendering so the judgement can be tested. The thresholds are
160 * about what a reader should DO: hours of quiet on a platform whose AI runs
161 * on CI failures is ordinary; weeks of it is the signature of a key that
162 * stopped working and a platform that never said so.
163 */
164export function describeLastSuccess(last: Date | null, now: Date = new Date()): {
165 tone: "ok" | "warn" | "bad";
166 text: string;
167} {
168 if (!last) {
169 return {
170 tone: "bad",
171 text: "No AI call has ever been billed on this instance — nothing has proven the key works.",
172 };
173 }
174 const days = (now.getTime() - last.getTime()) / 86_400_000;
175 const stamp = last.toISOString().slice(0, 16).replace("T", " ") + " UTC";
176 if (days < 2) return { tone: "ok", text: `Last billed call ${stamp}.` };
177 if (days < 7) {
178 return {
179 tone: "warn",
180 text: `Last billed call ${stamp} — ${Math.floor(days)} days ago. Quiet, but not yet unusual.`,
181 };
182 }
183 return {
184 tone: "bad",
185 text:
186 `Last billed call ${stamp} — ${Math.floor(days)} days ago. ` +
187 `On a platform whose AI runs on CI failures, that gap usually means the key stopped working, not that nothing needed it.`,
188 };
189}
Modifiedsrc/routes/admin.tsx+94−0View fileUnifiedSplit
@@ -51,7 +51,9 @@ import {
5151 AI_MODEL_FLAG_KEYS,
5252 AVAILABLE_MODELS,
5353 MODEL_SONNET,
54 aiOutage,
5455} from "../lib/ai-client";
56import { describeLastSuccess, probeProvider } from "../lib/ai-provider-health";
5557import { audit } from "../lib/notify";
5658import { sendDigestsToAll, sendDigestForUser } from "../lib/email-digest";
5759import {
@@ -4833,6 +4835,42 @@ admin.post("/admin/autopilot/run", async (c) => {
48334835
48344836// ─── AI Cost Breakdown (/admin/ai-costs) ─────────────────────────────────────
48354837
4838/**
4839 * Ask the provider, right now, whether this key works.
4840 *
4841 * POST because it spends money — a GET would be prefetched by a browser, a
4842 * link checker or a crawler, and "health check that bills on someone else's
4843 * schedule" is how a diagnostic becomes a cost.
4844 *
4845 * The answer comes back as redirect params rather than a rendered POST
4846 * response so that refreshing the page does not silently re-bill. The
4847 * provider's own words are passed through verbatim: "credit balance is too
4848 * low" and "invalid x-api-key" demand opposite actions, and the humanised
4849 * "AI unavailable" that the rest of the platform shows users is the wrong
4850 * text for the one person who can actually fix it.
4851 */
4852admin.post("/admin/ai-costs/probe", async (c) => {
4853 const g = await gate(c);
4854 if (g instanceof Response) return g;
4855 const probe = await probeProvider();
4856 await audit({
4857 userId: g.user.id,
4858 action: "admin.ai.probe",
4859 metadata: {
4860 ok: probe.ok,
4861 status: probe.status,
4862 detail: probe.detail.slice(0, 300),
4863 },
4864 }).catch(() => {});
4865 if (probe.ok) return c.redirect("/admin/ai-costs?probe=ok");
4866 const q = new URLSearchParams({
4867 probe_error: `${probe.status ? `${probe.status}: ` : ""}${probe.detail}`.slice(0, 400),
4868 });
4869 if (probe.advice) q.set("probe_advice", probe.advice.slice(0, 400));
4870 return c.redirect(`/admin/ai-costs?${q.toString()}`);
4871});
4872
4873
48364874admin.get("/admin/ai-costs", async (c) => {
48374875 const g = await gate(c);
48384876 if (g instanceof Response) return g;
@@ -4875,6 +4913,23 @@ admin.get("/admin/ai-costs", async (c) => {
48754913 .orderBy(sql`sum(${aiCostEvents.centsEstimate}) desc`)
48764914 .limit(10);
48774915
4916 // Provider health. The spend numbers below answer "what did we spend"; they
4917 // cannot answer "can we spend anything at all", and for the whole of August
4918 // the answer to the second was no while the first rendered a tidy $1.75.
4919 // A cost page that cannot show a dead key is a page that shows a healthy
4920 // month during an outage.
4921 const [lastCall] = await db
4922 .select({ at: aiCostEvents.occurredAt })
4923 .from(aiCostEvents)
4924 .orderBy(desc(aiCostEvents.occurredAt))
4925 .limit(1);
4926 const lastSuccess = describeLastSuccess(lastCall?.at ?? null, now);
4927 const outage = aiOutage();
4928 // Probe results arrive as a redirect param so a refresh does not re-bill.
4929 const probeOk = c.req.query("probe") === "ok";
4930 const probeErr = c.req.query("probe_error") ?? "";
4931 const probeAdvice = c.req.query("probe_advice") ?? "";
4932
48784933 const maxCents = Math.max(1, ...byCategory.map((r) => Number(r.cents)));
48794934 const maxSpenderCents = Math.max(1, ...topSpenders.map((r) => Number(r.cents)));
48804935
@@ -4906,6 +4961,45 @@ admin.get("/admin/ai-costs", async (c) => {
49064961 </div>
49074962 </section>
49084963
4964 <div
4965 class="adm-analytics-stat"
4966 style={`margin-bottom:18px;border-left:4px solid ${
4967 probeOk ? "#1f5f57" : lastSuccess.tone === "bad" || outage.active ? "#b4462f" : "#b8860b"
4968 }`}
4969 >
4970 <div class="adm-analytics-stat-label">Can we reach the model?</div>
4971 <div style="margin:6px 0 10px;font-size:14px;line-height:1.55">
4972 {probeOk ? (
4973 <strong>Live probe succeeded — the key works right now.</strong>
4974 ) : probeErr ? (
4975 <>
4976 <strong>Live probe failed.</strong> The provider said:{" "}
4977 <code style="word-break:break-word">{probeErr}</code>
4978 {probeAdvice ? <div style="margin-top:6px">{probeAdvice}</div> : null}
4979 </>
4980 ) : (
4981 <span style="opacity:.8">
4982 No probe run this page load. The evidence below exists either way.
4983 </span>
4984 )}
4985 </div>
4986 <div style="font-size:13px;line-height:1.55">{lastSuccess.text}</div>
4987 {outage.active ? (
4988 <div style="margin-top:8px;font-size:13px;line-height:1.55">
4989 <strong>Currently deferring AI work:</strong> {outage.reason}
4990 </div>
4991 ) : null}
4992 <form method="post" action="/admin/ai-costs/probe" style="margin-top:12px">
4993 <button type="submit" class="adm-btn">
4994 Run a live probe
4995 </button>
4996 <span style="margin-left:10px;font-size:12px;opacity:.7">
4997 One Haiku call, max_tokens 1 — about five millionths of a dollar.
4998 Never runs on page load.
4999 </span>
5000 </form>
5001 </div>
5002
49095003 <div class="adm-analytics-statgrid">
49105004 <div class="adm-analytics-stat">
49115005 <div class="adm-analytics-stat-label">Total spend this month</div>
49125006
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts