fix(spine): anthropic probe is billing-aware; remove tonight's duplicate spine #5457
5 changed files+43−475
Modifiedsrc/__tests__/dependency-probes.test.ts+12−4View fileUnifiedSplit
@@ -160,11 +160,14 @@ describe("runDependencyProbes", () => {
160160// ─── the real probe registry ─────────────────────────────────────────
161161
162162describe("HTTP_PROBES", () => {
163 it("only ever probes free, read-only endpoints", () => {
164 // A probe pointed at a billed endpoint would run forever on a timer.
165 // Keep this assertion honest if you add a probe.
163 it("only probes allowlisted endpoints, and billed ones are bounded", () => {
164 // The old rule was "free endpoints only" — and the free /v1/models
165 // probe stayed green through a full day of exhausted-balance outage
166 // (2026-08-08): a probe that bills nothing cannot see a billing
167 // failure. The anthropic probe is now a deliberate 1-output-token
168 // inference call; this test pins that its spend stays bounded.
166169 const allowed = [
167 "https://api.anthropic.com/v1/models",
170 "https://api.anthropic.com/v1/messages",
168171 "https://api.stripe.com/v1/balance",
169172 "https://api.resend.com/domains",
170173 ];
@@ -172,6 +175,11 @@ describe("HTTP_PROBES", () => {
172175 const resolved = probe.resolve();
173176 if (!resolved) continue; // unconfigured in this environment
174177 expect(allowed.some((a) => resolved.url.startsWith(a))).toBe(true);
178 if (resolved.url.includes("/v1/messages")) {
179 const body = JSON.parse(resolved.body || "{}");
180 expect(body.max_tokens).toBe(1);
181 expect(body.model).toContain("haiku");
182 }
175183 }
176184 });
177185
Modifiedsrc/app.tsx+0−3View fileUnifiedSplit
@@ -105,7 +105,6 @@ import orgRoutes from "./routes/orgs";
105105import notificationRoutes from "./routes/notifications";
106106import onboardingRoutes from "./routes/onboarding";
107107import buildingItselfRoutes from "./routes/building-itself";
108import spineRoutes from "./routes/spine";
109108import adminRoutes from "./routes/admin";
110109import adminDeletionsRoutes from "./routes/admin-deletions";
111110import adminStripeRoutes from "./routes/admin-stripe";
@@ -1037,8 +1036,6 @@ app.route("/", connectHubRoutes);
10371036// Building-itself proof page — STRATEGY-30 #1. Aggregate-only, public.
10381037// Mounted here (before webRoutes) so /:owner can't swallow the path.
10391038app.route("/", buildingItselfRoutes);
1040// The Spine — heartbeat board (/admin/spine + /api/spine). Site-admin only.
1041app.route("/", spineRoutes);
10421039// Push Watch — per-commit live status (gates + deploy + latency) at /:owner/:repo/push/:sha
10431040app.route("/", pushWatchRoutes);
10441041// Org Secrets Manager — BLOCK M2 — /orgs/:slug/settings/secrets
Modifiedsrc/lib/dependency-probes.ts+31−5View fileUnifiedSplit
@@ -54,8 +54,17 @@ export interface DependencyProbe {
5454 * Resolve the probe at call time. Returning `null` means "not
5555 * configured" and yields yellow. Reading env here (rather than at
5656 * module load) matches config.ts's getter pattern.
57 *
58 * `method`/`body` are optional (default GET, no body) so a probe can
59 * exercise the capability that actually matters — reachability probes
60 * stay green through billing failures.
5761 */
58 resolve: () => { url: string; headers: Record<string, string> } | null;
62 resolve: () => {
63 url: string;
64 headers: Record<string, string>;
65 method?: "GET" | "POST";
66 body?: string;
67 } | null;
5968}
6069
6170/** Check name for a probe. */
@@ -75,10 +84,26 @@ export const HTTP_PROBES: readonly DependencyProbe[] = [
7584 resolve: () => {
7685 const key = config.anthropicApiKey;
7786 if (!key) return null;
87 // A REAL 1-output-token inference call, not /v1/models. The models
88 // list "bills nothing" — which is exactly why it stayed green for a
89 // full day (2026-08-08) while the credit balance was exhausted and
90 // every AI feature on the platform was down. The only probe that
91 // detects a billing failure is one that gets billed. Cost: ~1 token
92 // on Haiku every 10 minutes — fractions of a cent per month; the
93 // blind spot cost a day.
7894 return {
79 // Cheapest authenticated read on the API. Bills nothing.
80 url: "https://api.anthropic.com/v1/models?limit=1",
81 headers: { "x-api-key": key, "anthropic-version": "2023-06-01" },
95 url: "https://api.anthropic.com/v1/messages",
96 method: "POST",
97 headers: {
98 "x-api-key": key,
99 "anthropic-version": "2023-06-01",
100 "content-type": "application/json",
101 },
102 body: JSON.stringify({
103 model: "claude-haiku-4-5",
104 max_tokens: 1,
105 messages: [{ role: "user", content: "ok" }],
106 }),
82107 };
83108 },
84109 },
@@ -175,8 +200,9 @@ async function runHttpProbe(
175200 const timer = setTimeout(() => controller.abort(), timeoutMs);
176201 try {
177202 const res = await fetchImpl(resolved.url, {
178 method: "GET",
203 method: resolved.method ?? "GET",
179204 headers: resolved.headers,
205 body: resolved.body,
180206 signal: controller.signal,
181207 });
182208 const outcome = classifyProbeResponse(res.status);
Deletedsrc/lib/spine.ts+0−357View fileUnifiedSplit
@@ -1,357 +0,0 @@
1/**
2 * The Spine — the platform's central nervous system.
3 *
4 * Owner directive (repeated ask, delivered 2026-08-08): one place that
5 * knows every vital sign and DETECTS HEARTBEAT FAILURES — instead of the
6 * owner discovering, via screenshots, that the AI balance died days ago,
7 * the mirror is two weeks stale, or a feed quietly stopped flowing.
8 *
9 * Design:
10 * - A registry of named heartbeats. Each check answers three things:
11 * ok?, what it measured, and why it failed (a human sentence, not an
12 * error envelope).
13 * - `runSpine()` executes all checks (each individually time-boxed and
14 * exception-proof), caches the result for CACHE_MS, and records
15 * STATE TRANSITIONS: a heartbeat flipping ok→failing writes a
16 * platform_errors row via reportError (so the existing error trail +
17 * admin surfaces light up) and remembers the flip in system_flags so
18 * it fires once per transition, not once per sweep.
19 * - Honesty rule: a check that cannot run reports "unknown", never "ok".
20 *
21 * Consumers: /admin/spine (page) and /api/spine (JSON — poll it from
22 * uptime monitors or a cron). Checks are cheap enough to run on request.
23 */
24
25import { sql, gte, desc } from "drizzle-orm";
26import { db } from "../db";
27import {
28 auditLog,
29 prComments,
30 pullRequests,
31 platformErrors,
32 repositories,
33} from "../db/schema";
34import { getBuildInfo } from "./build-info";
35import { isAiAvailable, getAnthropic, MODEL_HAIKU } from "./ai-client";
36import { getFlag, setFlag } from "./admin";
37import { reportError } from "./observability";
38
39export type HeartbeatStatus = "ok" | "failing" | "unknown";
40
41export interface Heartbeat {
42 name: string;
43 /** What this vital sign protects, in one sentence. */
44 watches: string;
45 status: HeartbeatStatus;
46 /** Human sentence: measurement when ok, cause when failing. */
47 detail: string;
48 /** ms the check took. */
49 tookMs: number;
50}
51
52export interface SpineReport {
53 checkedAt: string;
54 overall: HeartbeatStatus;
55 beats: Heartbeat[];
56}
57
58const CACHE_MS = 60_000;
59const CHECK_TIMEOUT_MS = 8_000;
60
61let _cache: SpineReport | null = null;
62let _cacheAt = 0;
63let _running: Promise<SpineReport> | null = null;
64
65/** Time-box + exception-proof a single check. */
66async function guard(
67 name: string,
68 watches: string,
69 fn: () => Promise<{ status: HeartbeatStatus; detail: string }>
70): Promise<Heartbeat> {
71 const t0 = Date.now();
72 try {
73 const result = await Promise.race([
74 fn(),
75 new Promise<{ status: HeartbeatStatus; detail: string }>((resolve) =>
76 setTimeout(
77 () =>
78 resolve({
79 status: "failing",
80 detail: `Check timed out after ${CHECK_TIMEOUT_MS / 1000}s`,
81 }),
82 CHECK_TIMEOUT_MS
83 )
84 ),
85 ]);
86 return { name, watches, ...result, tookMs: Date.now() - t0 };
87 } catch (err) {
88 return {
89 name,
90 watches,
91 status: "failing",
92 detail: err instanceof Error ? err.message.slice(0, 200) : "Check threw",
93 tookMs: Date.now() - t0,
94 };
95 }
96}
97
98// ─── The heartbeats ─────────────────────────────────────────────────────
99
100async function beatDatabase() {
101 return guard("database", "Every feature — nothing works without it", async () => {
102 await db.execute(sql`select 1`);
103 return { status: "ok" as const, detail: "Responding" };
104 });
105}
106
107async function beatAiProvider() {
108 return guard(
109 "ai_provider",
110 "AI reviews, repairs, spec-to-PR, chat — every customer AI feature",
111 async () => {
112 if (!isAiAvailable()) {
113 return {
114 status: "failing" as const,
115 detail: "ANTHROPIC_API_KEY is not configured",
116 };
117 }
118 // A real (minimal) inference call — the only probe that catches an
119 // exhausted balance. ~1 output token on the cheapest model; the cost
120 // of knowing is a rounding error, the cost of NOT knowing was a full
121 // day of silent platform-wide AI failure (2026-08-08).
122 try {
123 await getAnthropic().messages.create({
124 model: MODEL_HAIKU,
125 max_tokens: 1,
126 messages: [{ role: "user", content: "ok" }],
127 });
128 return { status: "ok" as const, detail: "Inference responding" };
129 } catch (err) {
130 const msg = err instanceof Error ? err.message : String(err);
131 if (msg.toLowerCase().includes("credit balance is too low")) {
132 return {
133 status: "failing" as const,
134 detail:
135 "Anthropic API balance exhausted — every AI feature is down. Top up at console.anthropic.com → Plans & Billing.",
136 };
137 }
138 return { status: "failing" as const, detail: msg.slice(0, 200) };
139 }
140 }
141 );
142}
143
144async function beatDeployFreshness() {
145 return guard(
146 "deploy_freshness",
147 "Merges actually reaching production (the 60s update timer)",
148 async () => {
149 const info = getBuildInfo();
150 const builtAt = info.builtAt ? new Date(info.builtAt) : null;
151 const [latestMerge] = await db
152 .select({ mergedAt: pullRequests.mergedAt })
153 .from(pullRequests)
154 .where(sql`${pullRequests.mergedAt} is not null`)
155 .orderBy(desc(pullRequests.mergedAt))
156 .limit(1);
157 if (!latestMerge?.mergedAt) {
158 return { status: "ok" as const, detail: "No merges yet" };
159 }
160 if (!builtAt) {
161 return {
162 status: "unknown" as const,
163 detail: "Build time not exposed — cannot compare against merges",
164 };
165 }
166 const lagMs = latestMerge.mergedAt.getTime() - builtAt.getTime();
167 if (lagMs > 15 * 60 * 1000) {
168 const mins = Math.round(lagMs / 60000);
169 return {
170 status: "failing" as const,
171 detail: `Latest merge is ${mins} min newer than the running build — the deploy timer may be stuck (check systemctl list-timers 'gluecron-*')`,
172 };
173 }
174 return {
175 status: "ok" as const,
176 detail:
177 lagMs > 0
178 ? `Deploy pending (merge ${Math.round(lagMs / 60000)} min ago)`
179 : "Running build is newer than the latest merge",
180 };
181 }
182 );
183}
184
185async function beatActivityFlow() {
186 return guard(
187 "activity_flow",
188 "Audit trail + feeds — silence here means events stopped recording",
189 async () => {
190 const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
191 const [row] = await db
192 .select({ n: sql<number>`count(*)::int` })
193 .from(auditLog)
194 .where(gte(auditLog.createdAt, dayAgo));
195 const n = row?.n ?? 0;
196 if (n === 0) {
197 return {
198 status: "failing" as const,
199 detail:
200 "Zero audit events in 24h on a platform with daily activity — event recording has likely stopped",
201 };
202 }
203 return { status: "ok" as const, detail: `${n} events in 24h` };
204 }
205 );
206}
207
208async function beatErrorSurge() {
209 return guard(
210 "error_rate",
211 "Server + client error trail (platform_errors)",
212 async () => {
213 const hourAgo = new Date(Date.now() - 60 * 60 * 1000);
214 const [row] = await db
215 .select({ n: sql<number>`count(*)::int` })
216 .from(platformErrors)
217 .where(
218 sql`${platformErrors.lastSeenAt} >= ${hourAgo} and ${platformErrors.resolvedAt} is null`
219 );
220 const n = row?.n ?? 0;
221 if (n >= 10) {
222 return {
223 status: "failing" as const,
224 detail: `${n} distinct unresolved errors active in the last hour — check /admin/errors`,
225 };
226 }
227 return {
228 status: "ok" as const,
229 detail: n === 0 ? "No active errors this hour" : `${n} active (below surge threshold)`,
230 };
231 }
232 );
233}
234
235async function beatGitStore() {
236 return guard(
237 "git_store",
238 "The bare repositories on disk — the actual product data",
239 async () => {
240 const [anyRepo] = await db
241 .select({ diskPath: repositories.diskPath })
242 .from(repositories)
243 .orderBy(desc(repositories.updatedAt))
244 .limit(1);
245 if (!anyRepo) return { status: "ok" as const, detail: "No repos yet" };
246 const exists = await Bun.file(`${anyRepo.diskPath}/HEAD`).exists();
247 if (!exists) {
248 return {
249 status: "failing" as const,
250 detail: `Most recently updated repo's bare dir is missing its HEAD (${anyRepo.diskPath}) — volume unmounted or path drift`,
251 };
252 }
253 return { status: "ok" as const, detail: "Bare store readable" };
254 }
255 );
256}
257
258async function beatAiReviewIntegrity() {
259 return guard(
260 "ai_review_integrity",
261 "Gates telling the truth — reviews that RUN, not silently skip",
262 async () => {
263 const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
264 const [tot] = await db
265 .select({ n: sql<number>`count(*)::int` })
266 .from(prComments)
267 .where(
268 sql`${prComments.isAiReview} = true and ${prComments.createdAt} >= ${dayAgo}`
269 );
270 const [unavail] = await db
271 .select({ n: sql<number>`count(*)::int` })
272 .from(prComments)
273 .where(
274 sql`${prComments.isAiReview} = true and ${prComments.createdAt} >= ${dayAgo} and ${prComments.body} like '%AI review unavailable%'`
275 );
276 const total = tot?.n ?? 0;
277 const failed = unavail?.n ?? 0;
278 if (total > 0 && failed / total > 0.5) {
279 return {
280 status: "failing" as const,
281 detail: `${failed}/${total} AI review attempts in 24h could not run — reviews are failing open platform-wide`,
282 };
283 }
284 return {
285 status: "ok" as const,
286 detail: total === 0 ? "No review attempts in 24h" : `${total - failed}/${total} review attempts ran`,
287 };
288 }
289 );
290}
291
292// ─── Runner ─────────────────────────────────────────────────────────────
293
294const TRANSITION_FLAG_PREFIX = "spine_state_";
295
296async function recordTransitions(beats: Heartbeat[]): Promise<void> {
297 for (const beat of beats) {
298 try {
299 const flagKey = `${TRANSITION_FLAG_PREFIX}${beat.name}`;
300 const prev = (await getFlag(flagKey)) || "ok";
301 if (beat.status === "failing" && prev !== "failing") {
302 // ok→failing: scream once, through the existing error trail.
303 reportError(new Error(`[spine] heartbeat FAILING: ${beat.name} — ${beat.detail}`), {
304 spine: beat.name,
305 watches: beat.watches,
306 });
307 }
308 if (beat.status !== "unknown" && beat.status !== prev) {
309 await setFlag(flagKey, beat.status, null);
310 }
311 } catch {
312 /* transition bookkeeping must never break the report */
313 }
314 }
315}
316
317export async function runSpine(force = false): Promise<SpineReport> {
318 if (!force && _cache && Date.now() - _cacheAt < CACHE_MS) return _cache;
319 if (_running) return _running;
320
321 _running = (async () => {
322 const beats = await Promise.all([
323 beatDatabase(),
324 beatAiProvider(),
325 beatDeployFreshness(),
326 beatActivityFlow(),
327 beatErrorSurge(),
328 beatGitStore(),
329 beatAiReviewIntegrity(),
330 ]);
331 const overall: HeartbeatStatus = beats.some((b) => b.status === "failing")
332 ? "failing"
333 : beats.some((b) => b.status === "unknown")
334 ? "unknown"
335 : "ok";
336 const report: SpineReport = {
337 checkedAt: new Date().toISOString(),
338 overall,
339 beats,
340 };
341 void recordTransitions(beats);
342 _cache = report;
343 _cacheAt = Date.now();
344 return report;
345 })().finally(() => {
346 _running = null;
347 });
348
349 return _running;
350}
351
352/** Test-only. */
353export function __resetSpineForTests(): void {
354 _cache = null;
355 _cacheAt = 0;
356 _running = null;
357}
Deletedsrc/routes/spine.tsx+0−106View fileUnifiedSplit
@@ -1,106 +0,0 @@
1/**
2 * /admin/spine — the heartbeat board, and /api/spine — the JSON vital
3 * signs for external monitors (poll it from a cron/uptime checker; a
4 * non-"ok" overall is the page-me signal).
5 *
6 * Site-admin only: the details name infrastructure (paths, providers).
7 */
8
9import { Hono } from "hono";
10import { Layout } from "../views/layout";
11import { requireAuth, softAuth } from "../middleware/auth";
12import type { AuthEnv } from "../middleware/auth";
13import { isSiteAdmin } from "../lib/admin";
14import { runSpine } from "../lib/spine";
15
16const spine = new Hono<AuthEnv>();
17spine.use("*", softAuth);
18
19const styles = `
20 .spn-wrap { max-width: 900px; margin: 0 auto; padding: var(--space-6) var(--space-4); }
21 .spn-head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; margin-bottom: var(--space-5); }
22 .spn-title { font-family: var(--font-display); font-size: 28px; font-weight: 800; letter-spacing: -0.02em; margin: 0; }
23 .spn-meta { font-family: var(--font-mono); font-size: 12px; color: var(--text-muted); }
24 .spn-overall {
25 display: inline-flex; align-items: center; gap: 8px;
26 padding: 6px 14px; border-radius: var(--r-full);
27 font-size: var(--t-sm); font-weight: 650; border: 1px solid;
28 }
29 .spn-overall.is-ok { color: var(--green); border-color: var(--green); }
30 .spn-overall.is-failing { color: var(--red); border-color: var(--red); }
31 .spn-overall.is-unknown { color: var(--yellow); border-color: var(--yellow); }
32 .spn-board { border: 1px solid var(--border); border-radius: var(--r-lg); background: var(--bg-elevated); overflow: hidden; margin-top: var(--space-4); }
33 .spn-beat { display: flex; gap: 14px; align-items: flex-start; padding: 14px var(--space-5); border-bottom: 1px solid var(--border-subtle); }
34 .spn-beat:last-child { border-bottom: none; }
35 .spn-dot { width: 10px; height: 10px; border-radius: 9999px; margin-top: 5px; flex-shrink: 0; }
36 .spn-dot.is-ok { background: var(--green); }
37 .spn-dot.is-failing { background: var(--red); box-shadow: 0 0 0 4px color-mix(in srgb, var(--red) 20%, transparent); }
38 .spn-dot.is-unknown { background: var(--yellow); }
39 .spn-beat-body { flex: 1; min-width: 0; }
40 .spn-beat-name { font-family: var(--font-mono); font-size: var(--t-base); font-weight: 650; }
41 .spn-beat-watches { font-size: var(--t-xs); color: var(--text-faint); margin-top: 1px; }
42 .spn-beat-detail { font-size: var(--t-sm); color: var(--text-muted); margin-top: 4px; }
43 .spn-beat.is-failing .spn-beat-detail { color: var(--red); }
44 .spn-took { font-family: var(--font-mono); font-size: 11px; color: var(--text-faint); flex-shrink: 0; margin-top: 5px; }
45 .spn-foot { margin-top: var(--space-4); font-size: var(--t-sm); color: var(--text-faint); }
46`;
47
48spine.get("/admin/spine", requireAuth, async (c) => {
49 const user = c.get("user")!;
50 if (!(await isSiteAdmin(user.id))) return c.notFound();
51
52 const report = await runSpine(c.req.query("force") === "1");
53
54 return c.html(
55 <Layout title="Spine — heartbeats" user={user}>
56 <style dangerouslySetInnerHTML={{ __html: styles }} />
57 <div class="spn-wrap">
58 <div class="spn-head">
59 <h1 class="spn-title">Spine</h1>
60 <span class={`spn-overall is-${report.overall}`}>
61 <span class={`spn-dot is-${report.overall}`} aria-hidden="true" />
62 {report.overall === "ok"
63 ? "All heartbeats OK"
64 : report.overall === "failing"
65 ? "Heartbeat failure"
66 : "Partially unknown"}
67 </span>
68 </div>
69 <div class="spn-meta">
70 Checked {new Date(report.checkedAt).toLocaleString()} ·{" "}
71 <a href="/admin/spine?force=1">re-check now</a> · JSON at{" "}
72 <code>/api/spine</code>
73 </div>
74 <div class="spn-board">
75 {report.beats.map((b) => (
76 <div class={`spn-beat is-${b.status}`}>
77 <span class={`spn-dot is-${b.status}`} aria-hidden="true" />
78 <div class="spn-beat-body">
79 <div class="spn-beat-name">{b.name}</div>
80 <div class="spn-beat-watches">{b.watches}</div>
81 <div class="spn-beat-detail">{b.detail}</div>
82 </div>
83 <span class="spn-took">{b.tookMs}ms</span>
84 </div>
85 ))}
86 </div>
87 <p class="spn-foot">
88 A heartbeat turning red files itself into the platform error trail
89 the moment it flips — failures announce themselves instead of
90 waiting to be discovered. Point an external monitor at{" "}
91 <code>/api/spine</code> and alert on any overall ≠ "ok".
92 </p>
93 </div>
94 </Layout>
95 );
96});
97
98spine.get("/api/spine", requireAuth, async (c) => {
99 const user = c.get("user")!;
100 if (!(await isSiteAdmin(user.id))) return c.json({ error: "Not found" }, 404);
101 const report = await runSpine(c.req.query("force") === "1");
102 c.header("cache-control", "no-store");
103 return c.json(report, report.overall === "failing" ? 503 : 200);
104});
105
106export default spine;
1070
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts