fix(ai): an outage is deferred, not judged as unfixable #5576
2 changed files+166−0
Modifiedsrc/__tests__/ai-ci-healer-diagnosis.test.ts+87−0View fileUnifiedSplit
@@ -115,3 +115,90 @@ describe("truncation cannot masquerade as 'unfixable'", () => {
115115 expect(Number(m![1])).toBeGreaterThanOrEqual(8000);
116116 });
117117});
118
119describe("an outage is deferred, not judged", () => {
120 /**
121 * INCIDENT 2026-08-30. The Anthropic account ran out of credit and every
122 * call returned:
123 *
124 * 400 invalid_request_error — "Your credit balance is too low to access
125 * the Anthropic API."
126 *
127 * The healer recorded each one as `ai.ci.gave_up` — the same permanent
128 * verdict it writes when the model has read the logs and judged a failure
129 * genuinely unfixable. hasMarker() skips marked runs forever, so topping up
130 * the account would not have retried a single one. A check that never ran,
131 * recorded as a check that came back negative.
132 */
133 const TRANSIENT = [
134 "ai_unavailable",
135 "client_init_failed",
136 "claude_call_failed",
137 "run_load_failed",
138 ] as const;
139
140 const TERMINAL = [
141 "unparseable_response",
142 "model_said_unfixable",
143 "no_usable_paths",
144 "run_not_failed",
145 ] as const;
146
147 it("splits on whether the model was billed, and the two sets are exhaustive", async () => {
148 const src = await Bun.file(
149 require("path").resolve("src/lib/ai-ci-healer.ts")
150 ).text();
151 // Anchor on `new Set([`, not the declaration: the type annotation
152 // contains AnalysisDiagnosis["reason"], whose `]` comes first.
153 const block = src.slice(src.indexOf("const TRANSIENT_DIAGNOSES"));
154 const arr = block.slice(block.indexOf("new Set(["));
155 const declared = arr.slice(0, arr.indexOf("]")).match(/"[a-z_]+"/g) ?? [];
156 const names = declared.map((q) => q.replace(/"/g, ""));
157
158 // Exactly the outcomes that never reached a billable response.
159 expect(new Set(names)).toEqual(new Set(TRANSIENT));
160
161 // Nothing that DID cost money may be retried freely — a bad response
162 // would otherwise re-bill every tick for the 24h a run stays eligible.
163 for (const t of TERMINAL) expect(names).not.toContain(t);
164
165 // And every reason in the union is classified one way or the other. If
166 // someone adds a ninth without deciding, this fails rather than letting
167 // it default to "permanent".
168 const union = src.slice(
169 src.indexOf("export type AnalysisDiagnosis"),
170 src.indexOf("export async function analyzeFailedWorkflowRun")
171 );
172 const all = [...new Set((union.match(/reason: "([a-z_]+)"/g) ?? []).map((m) =>
173 m.replace(/reason: "|"/g, "")
174 ))];
175 for (const r of all) {
176 expect([...TRANSIENT, ...TERMINAL]).toContain(r as never);
177 }
178 });
179
180 it("a credit-balance 400 is classified transient", () => {
181 // The literal production error, so the classification is anchored to a
182 // real payload rather than a category name someone might re-file later.
183 const d: AnalysisDiagnosis = {
184 reason: "claude_call_failed",
185 detail:
186 '400 {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Anthropic API."}}',
187 };
188 expect(TRANSIENT).toContain(d.reason as never);
189 });
190
191 it("deferral uses an action hasMarker does not honour", async () => {
192 // The mechanism the retry depends on: hasMarker matches only healed and
193 // gave_up. If ai.ci.deferred were ever added to that IN clause, deferred
194 // runs would be skipped forever and this fix would silently revert.
195 const src = await Bun.file(
196 require("path").resolve("src/lib/ai-ci-healer.ts")
197 ).text();
198 const marker = src.slice(src.indexOf("async function hasMarker"));
199 const inClause = marker.slice(0, 700);
200 expect(inClause).toContain("ai.ci.healed");
201 expect(inClause).toContain("ai.ci.gave_up");
202 expect(inClause).not.toContain("ai.ci.deferred");
203 });
204});
Modifiedsrc/lib/ai-ci-healer.ts+79−0View fileUnifiedSplit
@@ -281,6 +281,31 @@ export interface AnalyzeOptions {
281281 onDiagnosis?: (d: AnalysisDiagnosis) => void;
282282}
283283
284/**
285 * Diagnoses that must NOT write a permanent marker.
286 *
287 * INCIDENT 2026-08-30: the Anthropic account ran out of credit. Every call
288 * returned `400 invalid_request_error — Your credit balance is too low`, and
289 * the healer recorded each one as `ai.ci.gave_up` — the same permanent verdict
290 * it writes when the model has read the logs and judged a failure unfixable.
291 * hasMarker() then skips those runs forever, so restoring credit would NOT
292 * have retried a single one of them. A check that never ran was being recorded
293 * as a check that came back negative.
294 *
295 * The rule that decides membership is billing, and it is not arbitrary: these
296 * are exactly the outcomes where the model was never billed, so retrying is
297 * free. Anything that reached the model and produced an answer we could not
298 * use — `unparseable_response`, `model_said_unfixable`, `no_usable_paths` —
299 * cost money and stays marked, or a bad response would re-bill on every tick
300 * for the 24 hours a run stays inside the healer's window.
301 */
302const TRANSIENT_DIAGNOSES: ReadonlySet<AnalysisDiagnosis["reason"]> = new Set([
303 "ai_unavailable",
304 "client_init_failed",
305 "claude_call_failed",
306 "run_load_failed",
307]);
308
284309/** Why an analysis produced no usable result. */
285310export type AnalysisDiagnosis =
286311 | { reason: "ai_unavailable" }
@@ -495,6 +520,48 @@ export interface HealOneOptions extends AnalyzeOptions {
495520 generatePatch?: typeof generatePatchForGateTestFinding;
496521}
497522
523/**
524 * Record that a run was DEFERRED, not judged.
525 *
526 * Deliberately uses `ai.ci.deferred`, which hasMarker() does not match — the
527 * run stays eligible and is retried once the environment recovers. Written at
528 * most once per run so a multi-hour outage cannot flood the audit log at five
529 * runs a tick.
530 */
531async function noteDeferred(
532 runId: string,
533 repositoryId: string | null,
534 commitSha: string | null,
535 diagnosis: AnalysisDiagnosis,
536 detail: Record<string, unknown>
537): Promise<void> {
538 try {
539 const [existing] = await db
540 .select({ id: auditLog.id })
541 .from(auditLog)
542 .where(
543 and(
544 eq(auditLog.targetType, "workflow_run"),
545 eq(auditLog.targetId, runId),
546 eq(auditLog.action, "ai.ci.deferred")
547 )
548 )
549 .limit(1);
550 if (existing) return;
551 } catch {
552 // Can't check — fall through and write. A duplicate audit row is a far
553 // cheaper mistake than losing the only record of why healing stalled.
554 }
555 await audit({
556 userId: null,
557 repositoryId,
558 action: "ai.ci.deferred",
559 targetType: "workflow_run",
560 targetId: runId,
561 metadata: { commitSha, diagnosis: diagnosis.reason, ...detail },
562 });
563}
564
498565export interface HealOneResult {
499566 outcome: "healed" | "gave_up" | "skipped";
500567 prNumber?: number;
@@ -595,6 +662,18 @@ export async function healOneRun(
595662 string,
596663 unknown
597664 >;
665
666 // Environment failed, not the run. Leave NO permanent marker, so this run
667 // is picked up again once the environment recovers — the whole point of
668 // separating "we could not look" from "we looked and found nothing".
669 if (d && TRANSIENT_DIAGNOSES.has(d.reason)) {
670 console.warn(
671 `[ai-ci-healer] deferring run=${runId}: ${d.reason} — not marked, will retry`
672 );
673 await noteDeferred(runId, repositoryId, commitSha, d, diagnosisDetail);
674 return { outcome: "skipped", reason: d.reason };
675 }
676
598677 await audit({
599678 userId: null,
600679 repositoryId,
601680
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts