feat(ai): learn the provider is down once, instead of rediscovering it everywhere #5582
3 changed files+181−0
Addedsrc/__tests__/ai-outage-cache.test.ts+102−0View fileUnifiedSplit
@@ -0,0 +1,102 @@
1/**
2 * Knowing the provider is down, once, instead of rediscovering it everywhere.
3 *
4 * `isAiAvailable()` is a CONFIGURATION check: with a valid key and a zero
5 * balance it returns true, every guard in the codebase passes, and the 400
6 * lands separately in each caller. That is how one billing lapse on
7 * 2026-08-30 became a different-looking bug in a dozen subsystems — the CI
8 * healer logged 405 give-ups, required status checks blocked merges, and
9 * nothing anywhere said "the provider is down".
10 *
11 * These pin the two properties that make the cache safe to trust: it trips
12 * ONLY on failures that will repeat until a human acts, and it forgets on its
13 * own so a top-up recovers without a restart.
14 */
15
16import { describe, it, expect, beforeEach } from "bun:test";
17import { noteAiFailure, aiOutage, __resetAiOutage } from "../lib/ai-client";
18
19beforeEach(() => __resetAiOutage());
20
21describe("what counts as an outage", () => {
22 it("trips on the exact production error", () => {
23 // Verbatim from the 400 that stopped every AI feature today.
24 noteAiFailure(
25 new Error(
26 '400 {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits."}}'
27 )
28 );
29 const o = aiOutage();
30 expect(o.active).toBe(true);
31 // And says something a human can act on, not the raw envelope.
32 expect(o.reason.toLowerCase()).toContain("balance");
33 expect(o.reason).not.toContain("request_id");
34 });
35
36 it("trips on auth failures, which also repeat until someone acts", () => {
37 for (const msg of [
38 "401 authentication_error: invalid x-api-key",
39 "permission_error: your account is not authorised",
40 ]) {
41 __resetAiOutage();
42 noteAiFailure(new Error(msg));
43 expect(aiOutage().active).toBe(true);
44 }
45 });
46
47 it("does NOT trip on retryable failures", () => {
48 // The dangerous false positive: treating a rate limit or a blip as an
49 // outage would switch the AI tier off for ten minutes over a burst.
50 for (const msg of [
51 "429 rate_limit_error: too many requests",
52 "500 api_error: internal server error",
53 "529 overloaded_error",
54 "fetch failed: ECONNRESET",
55 "Request timed out",
56 ]) {
57 __resetAiOutage();
58 noteAiFailure(new Error(msg));
59 expect(aiOutage().active).toBe(false);
60 }
61 });
62
63 it("survives junk without throwing", () => {
64 // It is called from catch blocks. A diagnostic must never break the thing
65 // it is diagnosing.
66 for (const junk of [null, undefined, 42, {}, [], Symbol("x")]) {
67 expect(() => noteAiFailure(junk)).not.toThrow();
68 }
69 expect(aiOutage().active).toBe(false);
70 });
71});
72
73describe("recovery without intervention", () => {
74 it("reports inactive once expired, and reports nothing when clear", () => {
75 expect(aiOutage()).toEqual({ active: false, reason: "", until: 0 });
76 noteAiFailure(new Error("credit balance is too low"));
77 expect(aiOutage().until).toBeGreaterThan(Date.now());
78 // Bounded: a top-up must recover on its own, without a restart, and
79 // nothing may leave the AI tier off longer than the outage itself.
80 expect(aiOutage().until - Date.now()).toBeLessThanOrEqual(10 * 60_000);
81 });
82});
83
84describe("the healer honours it", () => {
85 it("short-circuits before spending a request, and defers rather than judges", async () => {
86 const src = await Bun.file(
87 require("path").resolve("src/lib/ai-ci-healer.ts")
88 ).text();
89 // Checked before the model call...
90 const outageIdx = src.indexOf("const outage = aiOutage();");
91 const callIdx = src.indexOf("client.messages.create");
92 expect(outageIdx).toBeGreaterThan(-1);
93 expect(outageIdx).toBeLessThan(callIdx);
94 // ...and reported as ai_unavailable, which TRANSIENT_DIAGNOSES treats as
95 // "retry later" — so a run is never permanently marked unhealable by an
96 // outage it had nothing to do with.
97 const block = src.slice(outageIdx, outageIdx + 400);
98 expect(block).toContain('reason: "ai_unavailable"');
99 // And the failure path feeds the cache, or nothing would ever trip it.
100 expect(src).toContain("noteAiFailure(err)");
101 });
102});
Modifiedsrc/lib/ai-ci-healer.ts+17−0View fileUnifiedSplit
@@ -39,9 +39,11 @@ import {
3939} from "../db/schema";
4040import {
4141 MODEL_SONNET,
42 aiOutage,
4243 extractText,
4344 getAnthropic,
4445 isAiAvailable,
46 noteAiFailure,
4547 parseJsonResponse,
4648} from "./ai-client";
4749import { audit } from "./notify";
@@ -342,6 +344,18 @@ export async function analyzeFailedWorkflowRun(
342344 opts.onDiagnosis?.({ reason: "ai_unavailable" });
343345 return null;
344346 }
347 // A known outage short-circuits before the request. isAiAvailable() only
348 // says a key is configured; this says the provider answered "no" to
349 // someone else moments ago and will answer "no" to us. Five runs a tick,
350 // every tick, is a lot of round trips to spend rediscovering that.
351 // Diagnosed as ai_unavailable, which is TRANSIENT — so the run keeps its
352 // place in the queue and is retried once the outage clears.
353 const outage = aiOutage();
354 if (outage.active) {
355 console.log(`[ai-ci-healer] skipping call — provider outage: ${outage.reason}`);
356 opts.onDiagnosis?.({ reason: "ai_unavailable" });
357 return null;
358 }
345359 try {
346360 client = getAnthropic();
347361 } catch {
@@ -451,6 +465,9 @@ export async function analyzeFailedWorkflowRun(
451465 "[ai-ci-healer] Claude call failed:",
452466 err instanceof Error ? err.message : err
453467 );
468 // Tell the rest of the platform, so the next caller does not have to
469 // rediscover an exhausted balance the expensive way.
470 noteAiFailure(err);
454471 opts.onDiagnosis?.({
455472 reason: "claude_call_failed",
456473 detail: err instanceof Error ? err.message : String(err),
Modifiedsrc/lib/ai-client.ts+62−0View fileUnifiedSplit
@@ -62,6 +62,68 @@ export function isAiAvailable(): boolean {
6262 return !!config.anthropicApiKey;
6363}
6464
65// ---------------------------------------------------------------------------
66// Provider outage cache
67//
68// isAiAvailable() above is a CONFIGURATION check, not a health check: with a
69// valid key and a zero balance it returns true, every guard in the codebase
70// passes, and the 400 lands separately in each caller. That is how a single
71// billing lapse became a different-looking bug in a dozen subsystems on
72// 2026-08-30 — the CI healer recorded 405 give-ups, the merge gate blocked
73// required checks, and nothing anywhere said "the provider is down".
74//
75// This records an ACCOUNT-LEVEL failure so callers can stop paying for the
76// round trip and can say the true thing instead. Deliberately narrow: only
77// failures that will repeat for every caller until a human acts. A 429 or a
78// 500 is retryable and must NOT trip it — treating a rate limit as an outage
79// would disable the AI tier for ten minutes over a burst.
80// ---------------------------------------------------------------------------
81
82const OUTAGE_TTL_MS = 10 * 60_000;
83let outageUntil = 0;
84let outageReason = "";
85
86/** Failures that will keep failing until someone tops up or fixes a key. */
87const ACCOUNT_LEVEL =
88 /credit balance is too low|insufficient[_ ]quota|billing|authentication_error|invalid x-api-key|permission_error/i;
89
90/**
91 * Record a provider failure. No-op unless it is account-level.
92 * Safe to call from any catch block; never throws.
93 */
94export function noteAiFailure(err: unknown): void {
95 try {
96 const msg =
97 err instanceof Error ? err.message : typeof err === "string" ? err : String(err);
98 if (!ACCOUNT_LEVEL.test(msg)) return;
99 outageUntil = Date.now() + OUTAGE_TTL_MS;
100 outageReason = humanizeAiError(msg);
101 } catch {
102 /* a diagnostic must never break the thing it is diagnosing */
103 }
104}
105
106/**
107 * Is the provider known-unavailable right now?
108 *
109 * Expires on its own so a top-up recovers without a restart — nothing here
110 * may leave the AI tier switched off longer than the outage itself.
111 */
112export function aiOutage(): { active: boolean; reason: string; until: number } {
113 const active = Date.now() < outageUntil;
114 return {
115 active,
116 reason: active ? outageReason : "",
117 until: active ? outageUntil : 0,
118 };
119}
120
121/** Test-only: forget any recorded outage. */
122export function __resetAiOutage(): void {
123 outageUntil = 0;
124 outageReason = "";
125}
126
65127/** Which backend is in use — for honest reporting on status surfaces. */
66128export function aiProviderName(): "anthropic" | "openai" {
67129 return config.aiProvider;
68130
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts