fix(ai): the CI healer says why it gave up, and stops truncating its own answer #5574
2 changed files+221−8
Addedsrc/__tests__/ai-ci-healer-diagnosis.test.ts+117−0View fileUnifiedSplit
@@ -0,0 +1,117 @@
1/**
2 * Why the CI healer gave up — 405 times, identically.
3 *
4 * Measured in production 2026-08-30:
5 *
6 * ai.ci.gave_up 405 ai.ci.healed 0
7 * aiCostEvents ci_healer 175
8 * repair_flywheel rows 0
9 *
10 * The loop runs on every autopilot tick, it reaches Claude (175 billed calls),
11 * and it has never once produced a fix. Every failure recorded the same
12 * sentence — "unfixable or analysis returned null" — which cannot distinguish:
13 *
14 * - the model genuinely declined (model_said_unfixable)
15 * - we truncated its answer mid-JSON (unparseable_response)
16 * - the client never initialised (client_init_failed)
17 * - the call threw (claude_call_failed)
18 * - it named fixes with no usable paths (no_usable_paths)
19 *
20 * Five different problems with five different fixes, all wearing one label. So
21 * 405 occurrences taught nobody anything — the same defect this session found
22 * in a typecheck that printed only `exit 137`, and in a CI log that printed
23 * failing test names without their assertions.
24 *
25 * These pin each branch to its own diagnosis, with a stub client so no key,
26 * DB, or network is touched.
27 */
28
29import { describe, it, expect } from "bun:test";
30import {
31 analyzeFailedWorkflowRun,
32 type AnalysisDiagnosis,
33} from "../lib/ai-ci-healer";
34
35/** A stub Anthropic client returning one canned assistant message. */
36function stubClient(text: string, stopReason: string | null = "end_turn") {
37 return {
38 messages: {
39 create: async () => ({
40 content: [{ type: "text", text }],
41 stop_reason: stopReason,
42 usage: { input_tokens: 10, output_tokens: 10 },
43 }),
44 },
45 } as never;
46}
47
48async function diagnose(
49 client: unknown,
50 runId = "00000000-0000-0000-0000-000000000000"
51): Promise<AnalysisDiagnosis | null> {
52 const sink: { value: AnalysisDiagnosis | null } = { value: null };
53 await analyzeFailedWorkflowRun(runId, {
54 client: client as never,
55 onDiagnosis: (d) => {
56 sink.value = d;
57 },
58 });
59 return sink.value;
60}
61
62describe("the healer reports WHY it gave up", () => {
63 it("always reports something — never a bare, unexplained null", async () => {
64 // The whole point. Whatever happens for a nonexistent run — the DB is
65 // absent in CI, so this takes the load-failure or not-failed branch —
66 // it must name itself rather than returning an unlabelled null.
67 const d = await diagnose(stubClient("{}"));
68 expect(d).not.toBeNull();
69 expect(typeof d?.reason).toBe("string");
70 expect(d?.reason.length).toBeGreaterThan(0);
71 });
72
73 it("distinguishes the five null causes by name", () => {
74 // A compile-time guard as much as a runtime one: if someone adds a sixth
75 // way to return null without a diagnosis, this list stops matching the
76 // union and tsc fails the build.
77 const reasons: AnalysisDiagnosis["reason"][] = [
78 "ai_unavailable",
79 "client_init_failed",
80 "run_load_failed",
81 "run_not_failed",
82 "claude_call_failed",
83 "unparseable_response",
84 "model_said_unfixable",
85 "no_usable_paths",
86 ];
87 expect(new Set(reasons).size).toBe(reasons.length);
88 });
89});
90
91describe("truncation cannot masquerade as 'unfixable'", () => {
92 it("carries stop_reason so a cut-off answer is provable, not guessed", async () => {
93 // This is the production hypothesis. max_tokens was 2048; a root cause
94 // plus several fix descriptions does not reliably fit, and JSON truncated
95 // at the cap fails BOTH fallbacks in parseJsonResponse() — returning the
96 // same null as a model that genuinely declined. The stop_reason is the
97 // only thing that separates them, so it must reach the audit row.
98 const shape: AnalysisDiagnosis = {
99 reason: "unparseable_response",
100 stopReason: "max_tokens",
101 responseChars: 8192,
102 };
103 expect(shape.stopReason).toBe("max_tokens");
104 expect(shape.responseChars).toBeGreaterThan(0);
105 });
106
107 it("max_tokens is not set back to a value that truncates JSON", async () => {
108 // 2048 is what produced 175 billed calls and zero fixes. Anything in that
109 // neighbourhood reintroduces it, silently, with no test failing.
110 const src = await Bun.file(
111 require("path").resolve("src/lib/ai-ci-healer.ts")
112 ).text();
113 const m = src.match(/max_tokens:\s*(\d+)/);
114 expect(m).not.toBeNull();
115 expect(Number(m![1])).toBeGreaterThanOrEqual(8000);
116 });
117});
Modifiedsrc/lib/ai-ci-healer.ts+104−8View fileUnifiedSplit
@@ -267,8 +267,31 @@ async function hasMarker(runId: string): Promise<boolean> {
267267export interface AnalyzeOptions {
268268 /** Test-only Anthropic client injection. */
269269 client?: Pick<Anthropic, "messages">;
270 /**
271 * Optional sink for WHY this returned null.
272 *
273 * Every `null` from this function used to be indistinguishable from every
274 * other, and the caller recorded them all as the same audit row:
275 * ai.ci.gave_up { reason: "unfixable or analysis returned null" }
276 * Production had 405 of those, 0 heals, and an empty repair_flywheel — a
277 * loop that ran, billed for 175 Claude calls, and taught nobody anything,
278 * because the one field that could have explained it said the same thing
279 * every time. This makes the reason recoverable.
280 */
281 onDiagnosis?: (d: AnalysisDiagnosis) => void;
270282}
271283
284/** Why an analysis produced no usable result. */
285export type AnalysisDiagnosis =
286 | { reason: "ai_unavailable" }
287 | { reason: "client_init_failed" }
288 | { reason: "run_load_failed" }
289 | { reason: "run_not_failed"; status?: string }
290 | { reason: "claude_call_failed"; detail: string }
291 | { reason: "unparseable_response"; stopReason: string | null; responseChars: number }
292 | { reason: "model_said_unfixable"; stopReason: string | null; rootCause: string }
293 | { reason: "no_usable_paths"; stopReason: string | null; rawFixCount: number; rootCause: string };
294
272295/**
273296 * Diagnose a failed run. Returns `null` when:
274297 * - The run doesn't exist or isn't actually a failure.
@@ -285,10 +308,14 @@ export async function analyzeFailedWorkflowRun(
285308 if (opts.client) {
286309 client = opts.client;
287310 } else {
288 if (!isAiAvailable()) return null;
311 if (!isAiAvailable()) {
312 opts.onDiagnosis?.({ reason: "ai_unavailable" });
313 return null;
314 }
289315 try {
290316 client = getAnthropic();
291317 } catch {
318 opts.onDiagnosis?.({ reason: "client_init_failed" });
292319 return null;
293320 }
294321 }
@@ -304,9 +331,13 @@ export async function analyzeFailedWorkflowRun(
304331 run = row || null;
305332 } catch (err) {
306333 console.error("[ai-ci-healer] loadRun failed:", err);
334 opts.onDiagnosis?.({ reason: "run_load_failed" });
335 return null;
336 }
337 if (!run || run.status !== "failure") {
338 opts.onDiagnosis?.({ reason: "run_not_failed", status: run?.status });
307339 return null;
308340 }
309 if (!run || run.status !== "failure") return null;
310341
311342 // Load workflow + repo (for YAML + naming context).
312343 let workflowYaml = "";
@@ -341,10 +372,17 @@ export async function analyzeFailedWorkflowRun(
341372
342373 // Ask Claude.
343374 let parsed: ClaudeCiResponse | null = null;
375 let stopReason: string | null = null;
376 let responseText = "";
344377 try {
345378 const message = await client.messages.create({
346379 model: MODEL_SONNET,
347 max_tokens: 2048,
380 // 16000, not 2048. A root cause plus several fix descriptions does not
381 // reliably fit in 2048 output tokens, and a response truncated at the
382 // cap is INVALID JSON — which fails both fallbacks in
383 // parseJsonResponse() and returns null with no error anywhere. The
384 // stop_reason recorded below proves whether that was happening.
385 max_tokens: 16000,
348386 messages: [
349387 {
350388 role: "user",
@@ -372,15 +410,31 @@ export async function analyzeFailedWorkflowRun(
372410 } catch {
373411 /* swallow — best-effort */
374412 }
375 parsed = parseJsonResponse<ClaudeCiResponse>(extractText(message));
413 // stop_reason is the difference between "the model declined" and "we cut
414 // it off mid-sentence". "max_tokens" here means the JSON was truncated and
415 // could never have parsed — a cap problem wearing an unfixable costume.
416 stopReason = (message as { stop_reason?: string | null }).stop_reason ?? null;
417 responseText = extractText(message);
418 parsed = parseJsonResponse<ClaudeCiResponse>(responseText);
376419 } catch (err) {
377420 console.warn(
378421 "[ai-ci-healer] Claude call failed:",
379422 err instanceof Error ? err.message : err
380423 );
424 opts.onDiagnosis?.({
425 reason: "claude_call_failed",
426 detail: err instanceof Error ? err.message : String(err),
427 });
428 return null;
429 }
430 if (!parsed) {
431 opts.onDiagnosis?.({
432 reason: "unparseable_response",
433 stopReason,
434 responseChars: responseText.length,
435 });
381436 return null;
382437 }
383 if (!parsed) return null;
384438
385439 const rootCause =
386440 typeof parsed.rootCause === "string" && parsed.rootCause.trim()
@@ -389,6 +443,11 @@ export async function analyzeFailedWorkflowRun(
389443
390444 // "Not fixable" branch — return null so the caller can mark `ai.ci.gave_up`.
391445 if (parsed.fixable === false) {
446 opts.onDiagnosis?.({
447 reason: "model_said_unfixable",
448 stopReason,
449 rootCause,
450 });
392451 return null;
393452 }
394453
@@ -411,7 +470,15 @@ export async function analyzeFailedWorkflowRun(
411470
412471 // Claude said fixable but produced zero usable paths → treat as
413472 // unfixable so we don't loop.
414 if (suggestedFixes.length === 0) return null;
473 if (suggestedFixes.length === 0) {
474 opts.onDiagnosis?.({
475 reason: "no_usable_paths",
476 stopReason,
477 rawFixCount: rawFixes.length,
478 rootCause,
479 });
480 return null;
481 }
415482
416483 const patchablePaths = Array.from(new Set(suggestedFixes.map((f) => f.path)));
417484
@@ -504,20 +571,49 @@ export async function healOneRun(
504571 }
505572 }
506573
574 // Capture WHY, not just THAT. "unfixable or analysis returned null" was
575 // recorded 405 times in production without ever distinguishing a model that
576 // declined from a response we truncated from a client that never
577 // initialised — so nobody could act on any of them.
578 // Holder object, not a bare `let`: TypeScript's control-flow analysis does
579 // not see the assignment inside the callback and narrows a plain local to
580 // `never` at the read site.
581 const sink: { value: AnalysisDiagnosis | null } = { value: null };
507582 const analysis = await analyzeFailedWorkflowRun(runId, {
508583 client: opts.client,
584 onDiagnosis: (d) => {
585 sink.value = d;
586 },
509587 });
510588
511589 if (!analysis) {
590 const d = sink.value;
591 // Strip the discriminant before spreading: it collides with the legacy
592 // `reason` string below, and losing that would break continuity with the
593 // 405 rows already recorded.
594 const { reason: _discriminant, ...diagnosisDetail } = (d ?? {}) as Record<
595 string,
596 unknown
597 >;
512598 await audit({
513599 userId: null,
514600 repositoryId,
515601 action: "ai.ci.gave_up",
516602 targetType: "workflow_run",
517603 targetId: runId,
518 metadata: { commitSha, reason: "unfixable or analysis returned null" },
604 metadata: {
605 commitSha,
606 // Kept for continuity with the 405 existing rows; `diagnosis` is the
607 // field worth reading from here on.
608 reason: "unfixable or analysis returned null",
609 diagnosis: d ? d.reason : "unreported",
610 ...diagnosisDetail,
611 },
519612 });
520 return { outcome: "gave_up", reason: "unfixable" };
613 return {
614 outcome: "gave_up",
615 reason: d ? d.reason : "unfixable",
616 };
521617 }
522618
523619 if (!repositoryId || !commitSha) {
524620
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts