ccantynz/Gluecron.com
CodeIssuesPull RequestsActionsSecurityInsights
✨ AI
Understand
Explain codebaseAsk AIChat with repoClaude sessionsNL searchSemantic searchArchaeologyDebt mapDocs tracking
Build
Spec to PRWorkspaceGenerate testsPropose migrationDev environmentCodebase migrator
Ship
Preview URLsAI release notesAI changelog
More
Repository
CommitsBranchesTagsReleasesPackagesSymbolsStale branches
Planning
MilestonesProjectsDiscussionsWikiContributors
Delivery
GatesMerge queueMerge checksDeploymentsPipelineAgents
Activity
Health reportPulsePush log
Settings

feat(medic): repair registry + spine doctor — recovery that proves itself #5536

Merged⚡ AI-generatedXSccantynz wants to mergefeat/medic-spine-doctor→mainopened 6d ago
9 changed files+1916−1
Addeddrizzle/0129_medic_outcomes.sql+32−0View fileUnifiedSplit
1-- medic_outcomes — the medic's ledger (2026-08-26, Mission Control §6).
2--
3-- One row per medic decision about a red check: an attempt (outcome
4-- 'repaired' or 'attempted-still-red') or a refusal (outcome 'refused'
5-- with a machine-readable reason). The Vapron medic contract this
6-- implements: a repair counts as recovered ONLY when the red check re-ran
7-- green (reverified_green), and a medic that quietly declines looks
8-- identical to a broken medic — so refusals are recorded outcomes, not
9-- silence.
10--
11-- The cooldown and per-day budget guards in src/lib/medic.ts are read
12-- FROM this table rather than from memory: the 60s deploy timer restarts
13-- the process every time main moves, and an in-memory budget would reset
14-- with it (the same argument remediation.ts makes for counting attempts
15-- from audit_log).
16--
17-- No FK anywhere: check_name references synthetic_checks.check_name by
18-- value, which is a name, not a row.
19
20--> statement-breakpoint
21CREATE TABLE IF NOT EXISTS "medic_outcomes" (
22 "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
23 "check_name" text NOT NULL,
24 "action" text NOT NULL,
25 "outcome" text NOT NULL,
26 "reason" text,
27 "detail" text,
28 "attempted_at" timestamp with time zone DEFAULT now() NOT NULL,
29 "reverified_green" boolean DEFAULT false NOT NULL
30);
31--> statement-breakpoint
32CREATE INDEX IF NOT EXISTS "medic_outcomes_check_attempted_idx" ON "medic_outcomes" ("check_name","attempted_at");
Addedscripts/design-render-sweep.mjs+122−0View fileUnifiedSplit
1// Design-render sweep — executes the manifest from design-render-manifest.mjs.
2//
3// Renders every page x viewport x theme to PNG. Resumable: existing files
4// are skipped, so a crashed run continues where it stopped. Writes
5// progress to sweep-log.txt and a machine-readable summary to
6// sweep-summary.json in the output dir.
7//
8// Usage:
9// node scripts/design-render-sweep.mjs --out <dir> [--manifest <file>]
10// (no manifest file -> generates in-process from the surface registry)
11//
12// Designed to run OUTSIDE the agent harness's process tree (Task
13// Scheduler, a detached Start-Process, or a plain interactive shell):
14// Chromium's multi-process launch dies inside constrained job objects —
15// discovered 2026-08-25 when every in-harness launch timed out while the
16// owner's own tabs rendered fine.
17
18import { chromium } from 'playwright';
19import { mkdirSync, existsSync, appendFileSync, writeFileSync, readFileSync } from 'fs';
20import { join } from 'path';
21
22const arg = (name, fallback) => {
23 const i = process.argv.indexOf(`--${name}`);
24 return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
25};
26
27const OUT = arg('out', join(process.cwd(), 'design-renders'));
28mkdirSync(OUT, { recursive: true });
29const LOG = join(OUT, 'sweep-log.txt');
30const log = (m) => {
31 const line = `${new Date().toISOString()} ${m}`;
32 console.log(line);
33 appendFileSync(LOG, line + '\n');
34};
35
36let manifest;
37const manifestFile = arg('manifest', null);
38if (manifestFile) {
39 manifest = JSON.parse(readFileSync(manifestFile, 'utf8'));
40} else {
41 const { surfaceSpecs } = await import('../src/lib/surface-monitor.ts');
42 const specs = surfaceSpecs({ repo: 'ccantynz/Gluecron.com' });
43 manifest = {
44 base: 'https://gluecron.com',
45 viewports: [
46 { name: 'desktop', width: 1440, height: 900 },
47 { name: 'tablet', width: 768, height: 1024 },
48 { name: 'mobile', width: 390, height: 844 },
49 ],
50 themes: [
51 { name: 'light', cookie: null },
52 { name: 'dark', cookie: 'theme=dark' },
53 ],
54 pages: [...new Set(specs.map((s) => s.path))].sort(),
55 };
56}
57
58const slug = (p) => (p === '/' ? 'home' : p.replace(/^\//, '').replace(/[^a-zA-Z0-9]+/g, '-').slice(0, 80));
59
60const jobs = [];
61for (const page of manifest.pages)
62 for (const vp of manifest.viewports)
63 for (const theme of manifest.themes) {
64 const file = join(OUT, `${slug(page)}--${vp.name}--${theme.name}.png`);
65 if (!existsSync(file)) jobs.push({ page, vp, theme, file });
66 }
67
68log(`sweep: ${jobs.length} renders to do (${manifest.pages.length} pages; already-done skipped)`);
69
70// Transport findings (2026-08-26, this machine): endpoint protection
71// blocks the browser AUTOMATION PIPE (--remote-debugging-pipe) globally —
72// bundled headless shell, real Edge, headed or headless, any session,
73// even Task Scheduler: all hang at launch. CDP over TCP is NOT blocked.
74// So: --cdp http://127.0.0.1:9222 connects to a browser someone launched
75// with --remote-debugging-port; --channel/--headed remain for machines
76// where normal launching works.
77const cdp = arg('cdp', null);
78const channel = arg('channel', null);
79const browser = cdp
80 ? await chromium.connectOverCDP(cdp)
81 : await chromium.launch({
82 headless: !process.argv.includes('--headed'),
83 ...(channel ? { channel } : {}),
84 });
85const failures = [];
86let done = 0;
87
88const CONCURRENCY = 3;
89async function worker() {
90 for (;;) {
91 const job = jobs.shift();
92 if (!job) return;
93 try {
94 const ctx = await browser.newContext({
95 viewport: { width: job.vp.width, height: job.vp.height },
96 });
97 if (job.theme.cookie) {
98 const [name, value] = job.theme.cookie.split('=');
99 await ctx.addCookies([{ name, value, domain: 'gluecron.com', path: '/' }]);
100 }
101 const p = await ctx.newPage();
102 await p.goto(manifest.base + job.page, { waitUntil: 'networkidle', timeout: 30000 });
103 await p.waitForTimeout(300);
104 await p.screenshot({ path: job.file, fullPage: true });
105 await ctx.close();
106 done++;
107 if (done % 25 === 0) log(`progress: ${done} done, ${jobs.length} left`);
108 } catch (e) {
109 failures.push({ ...job, error: String(e).slice(0, 200) });
110 log(`FAIL ${job.page} ${job.vp.name} ${job.theme.name}: ${String(e).slice(0, 120)}`);
111 }
112 }
113}
114await Promise.all(Array.from({ length: CONCURRENCY }, worker));
115await browser.close();
116
117writeFileSync(
118 join(OUT, 'sweep-summary.json'),
119 JSON.stringify({ finishedAt: new Date().toISOString(), rendered: done, failures }, null, 2)
120);
121writeFileSync(join(OUT, 'DONE.marker'), new Date().toISOString());
122log(`sweep complete: ${done} rendered, ${failures.length} failures`);
Addedsrc/__tests__/medic.test.ts+440−0View fileUnifiedSplit
1/**
2 * Medic + spine doctor tests — the Vapron medic contract itself.
3 *
4 * The properties pinned hardest, in order of how expensive their loss
5 * would be:
6 *
7 * - an action whose re-verify stays red MUST NOT record `repaired` —
8 * a restart exiting 0 proves nothing, and a medic that takes credit
9 * for unverified repairs is worse than no medic;
10 * - refusals are recorded outcomes with machine-readable reasons — a
11 * medic that quietly declines looks identical to a broken medic;
12 * - cooldown and budget guards actually stop attempts;
13 * - the spine doctor's age math, against an injected clock;
14 * - registry hygiene: every action comes from the closed allowlist.
15 *
16 * Everything is injected; no DB, no network.
17 */
18
19import { describe, it, expect } from "bun:test";
20import {
21 DOCTOR_MAX_AGES,
22 MEDIC_ACTION_NAMES,
23 REFUSAL_REPEAT_WINDOW_MS,
24 REPAIR_REGISTRY,
25 ruleFor,
26 runMedic,
27 spineDoctorChecks,
28 type MedicDeps,
29 type MedicHistoryRow,
30 type MedicRule,
31} from "../lib/medic";
32import type { SyntheticCheckResult } from "../lib/synthetic-monitor";
33
34const red = (name: string): SyntheticCheckResult => ({
35 name,
36 status: "red",
37 statusCode: 500,
38 durationMs: 10,
39 error: "HTTP 500",
40});
41
42const green = (name: string): SyntheticCheckResult => ({
43 name,
44 status: "green",
45 statusCode: 200,
46 durationMs: 5,
47});
48
49/** A rule that maps `target` to a real (non-none) action. */
50const testRule = (over: Partial<MedicRule> = {}): MedicRule => ({
51 check: "target",
52 action: "regenerate-check",
53 cooldownMs: 10 * 60_000,
54 maxPerDay: 3,
55 note: "test rule",
56 ...over,
57});
58
59/** Harness capturing persisted rows instead of writing them. */
60function harness(over: Partial<MedicDeps> = {}) {
61 const persisted: any[] = [];
62 const deps: MedicDeps = {
63 now: () => 1_000_000_000,
64 rules: [testRule()],
65 history: async () => [],
66 execute: async () => "did the thing",
67 reverify: async () => null,
68 persist: async (row) => {
69 persisted.push(row);
70 },
71 ...over,
72 };
73 return { deps, persisted };
74}
75
76describe("the contract: repaired requires a green re-run", () => {
77 it("NEVER records `repaired` when the re-verify stays red", async () => {
78 const { deps, persisted } = harness({
79 execute: async () => "restart exited 0", // "success" that proves nothing
80 reverify: async () => red("target"),
81 });
82 const s = await runMedic([red("target")], deps);
83 expect(s.repaired).toBe(0);
84 expect(s.attemptedStillRed).toBe(1);
85 expect(s.results[0]!.outcome).toBe("attempted-still-red");
86 expect(s.results[0]!.reverifiedGreen).toBe(false);
87 expect(persisted[0].outcome).toBe("attempted-still-red");
88 expect(persisted[0].reverifiedGreen).toBe(false);
89 });
90
91 it("a yellow re-verify is not green — still not repaired", async () => {
92 const { deps } = harness({
93 reverify: async () => ({
94 name: "target",
95 status: "yellow",
96 durationMs: 5,
97 }),
98 });
99 const s = await runMedic([red("target")], deps);
100 expect(s.results[0]!.outcome).toBe("attempted-still-red");
101 expect(s.results[0]!.reverifiedGreen).toBe(false);
102 });
103
104 it("an unavailable re-verify is not green — still not repaired", async () => {
105 const { deps } = harness({ reverify: async () => null });
106 const s = await runMedic([red("target")], deps);
107 expect(s.results[0]!.outcome).toBe("attempted-still-red");
108 expect(s.results[0]!.detail).toContain("cannot be re-verified");
109 });
110
111 it("records `repaired` with reverifiedGreen ONLY on a green re-run", async () => {
112 const { deps, persisted } = harness({
113 reverify: async () => green("target"),
114 });
115 const s = await runMedic([red("target")], deps);
116 expect(s.repaired).toBe(1);
117 expect(s.results[0]!.outcome).toBe("repaired");
118 expect(s.results[0]!.reverifiedGreen).toBe(true);
119 expect(persisted[0].reverifiedGreen).toBe(true);
120 });
121
122 it("an action that throws is attempted-still-red, not repaired, not a crash", async () => {
123 const { deps, persisted } = harness({
124 execute: async () => {
125 throw new Error("boom");
126 },
127 // Even a lying reverify can't rescue a thrown action: reverify is
128 // skipped after a throw, so 'repaired' is unreachable.
129 reverify: async () => green("target"),
130 });
131 const s = await runMedic([red("target")], deps);
132 expect(s.repaired).toBe(0);
133 expect(s.results[0]!.outcome).toBe("attempted-still-red");
134 expect(s.results[0]!.detail).toContain("action threw: boom");
135 expect(persisted).toHaveLength(1);
136 });
137});
138
139describe("refusals are recorded outcomes with reasons", () => {
140 it("a check with no rule refuses with reason no-rule", async () => {
141 const { deps, persisted } = harness({ rules: [] });
142 const s = await runMedic([red("mystery-check")], deps);
143 expect(s.refused).toBe(1);
144 expect(s.results[0]!.reason).toBe("no-rule");
145 expect(persisted[0].outcome).toBe("refused");
146 expect(persisted[0].reason).toBe("no-rule");
147 });
148
149 it("a `none` rule refuses with reason action-unsafe and never executes", async () => {
150 let executed = false;
151 const { deps, persisted } = harness({
152 rules: [testRule({ action: "none" })],
153 execute: async () => {
154 executed = true;
155 return "must not run";
156 },
157 });
158 const s = await runMedic([red("target")], deps);
159 expect(s.results[0]!.outcome).toBe("refused");
160 expect(s.results[0]!.reason).toBe("action-unsafe");
161 expect(executed).toBe(false);
162 expect(persisted).toHaveLength(1);
163 });
164
165 it("an unreadable ledger refuses (budget) rather than trying blind", async () => {
166 const { deps } = harness({
167 history: async () => {
168 throw new Error("db down");
169 },
170 });
171 const s = await runMedic([red("target")], deps);
172 expect(s.results[0]!.outcome).toBe("refused");
173 expect(s.results[0]!.reason).toBe("budget");
174 });
175
176 it("identical repeat refusals inside the window are not re-persisted", async () => {
177 const nowMs = 1_000_000_000;
178 const priorRefusal: MedicHistoryRow = {
179 outcome: "refused",
180 reason: "action-unsafe",
181 attemptedAt: new Date(nowMs - REFUSAL_REPEAT_WINDOW_MS / 2),
182 };
183 const { deps, persisted } = harness({
184 now: () => nowMs,
185 rules: [testRule({ action: "none" })],
186 history: async () => [priorRefusal],
187 });
188 const s = await runMedic([red("target")], deps);
189 // Still REPORTED as refused — only the duplicate row is suppressed.
190 expect(s.results[0]!.outcome).toBe("refused");
191 expect(persisted).toHaveLength(0);
192 });
193});
194
195describe("cooldown and budget guards", () => {
196 const nowMs = 1_000_000_000;
197 const attemptAt = (agoMs: number): MedicHistoryRow => ({
198 outcome: "attempted-still-red",
199 reason: null,
200 attemptedAt: new Date(nowMs - agoMs),
201 });
202
203 it("refuses with reason cooldown when the last attempt is too recent", async () => {
204 let executed = false;
205 const { deps } = harness({
206 now: () => nowMs,
207 rules: [testRule({ cooldownMs: 30 * 60_000 })],
208 history: async () => [attemptAt(60_000)], // 1 min ago, cooldown 30 min
209 execute: async () => {
210 executed = true;
211 return "must not run";
212 },
213 });
214 const s = await runMedic([red("target")], deps);
215 expect(s.results[0]!.outcome).toBe("refused");
216 expect(s.results[0]!.reason).toBe("cooldown");
217 expect(executed).toBe(false);
218 });
219
220 it("allows an attempt once the cooldown has passed", async () => {
221 const { deps } = harness({
222 now: () => nowMs,
223 rules: [testRule({ cooldownMs: 30 * 60_000, maxPerDay: 5 })],
224 history: async () => [attemptAt(31 * 60_000)],
225 reverify: async () => green("target"),
226 });
227 const s = await runMedic([red("target")], deps);
228 expect(s.results[0]!.outcome).toBe("repaired");
229 });
230
231 it("refuses with reason budget once maxPerDay attempts exist in 24h", async () => {
232 const { deps } = harness({
233 now: () => nowMs,
234 rules: [testRule({ cooldownMs: 60_000, maxPerDay: 3 })],
235 history: async () => [
236 attemptAt(2 * 60 * 60_000),
237 attemptAt(5 * 60 * 60_000),
238 attemptAt(9 * 60 * 60_000),
239 ],
240 });
241 const s = await runMedic([red("target")], deps);
242 expect(s.results[0]!.outcome).toBe("refused");
243 expect(s.results[0]!.reason).toBe("budget");
244 });
245
246 it("refusal rows do not consume attempt budget", async () => {
247 const refusedRow: MedicHistoryRow = {
248 outcome: "refused",
249 reason: "cooldown",
250 attemptedAt: new Date(nowMs - 60_000),
251 };
252 const { deps } = harness({
253 now: () => nowMs,
254 rules: [testRule({ cooldownMs: 30 * 60_000, maxPerDay: 1 })],
255 history: async () => [refusedRow, refusedRow, refusedRow],
256 reverify: async () => green("target"),
257 });
258 const s = await runMedic([red("target")], deps);
259 expect(s.results[0]!.outcome).toBe("repaired");
260 });
261});
262
263describe("runMedic never throws and only acts on red", () => {
264 it("survives a persist failure", async () => {
265 const { deps } = harness({
266 persist: async () => {
267 throw new Error("insert failed");
268 },
269 reverify: async () => green("target"),
270 });
271 const s = await runMedic([red("target")], deps);
272 expect(s.results[0]!.outcome).toBe("repaired");
273 });
274
275 it("skips non-red inputs entirely", async () => {
276 const { deps, persisted } = harness();
277 const s = await runMedic(
278 [green("target"), { name: "target", status: "yellow", durationMs: 1 }],
279 deps
280 );
281 expect(s.considered).toBe(0);
282 expect(persisted).toHaveLength(0);
283 });
284});
285
286describe("spine doctor age math (injected clock)", () => {
287 const nowMs = 10_000_000_000;
288 const latestWith = (rows: Array<{ name: string; ageMs: number }>) => {
289 const out: Record<string, { name: string; checkedAt: Date }> = {};
290 for (const r of rows) {
291 out[r.name] = { name: r.name, checkedAt: new Date(nowMs - r.ageMs) };
292 }
293 return out;
294 };
295
296 it("all green when every watcher's newest row is fresh", async () => {
297 const results = await spineDoctorChecks({
298 now: () => nowMs,
299 lastTickFinishedAt: () => new Date(nowMs - 2 * 60_000).toISOString(),
300 latest: async () =>
301 latestWith([
302 { name: "healthz", ageMs: 3 * 60_000 },
303 { name: "journey:login-form", ageMs: 4 * 60_000 },
304 { name: "surface:/admin", ageMs: 30 * 60_000 }, // < 60m budget
305 ]),
306 });
307 expect(results).toHaveLength(4);
308 for (const r of results) {
309 expect(r.status).toBe("green");
310 // The age lives in the detail, per the contract.
311 expect(r.error).toContain("old (max");
312 }
313 });
314
315 it("goes red exactly when a watcher's newest row exceeds its max age", async () => {
316 const results = await spineDoctorChecks({
317 now: () => nowMs,
318 lastTickFinishedAt: () => new Date(nowMs - 60_000).toISOString(),
319 latest: async () =>
320 latestWith([
321 { name: "healthz", ageMs: 16 * 60_000 }, // > 15m → red
322 { name: "journey:login-form", ageMs: 14 * 60_000 }, // < 15m → green
323 { name: "surface:/admin", ageMs: 61 * 60_000 }, // > 60m → red
324 ]),
325 });
326 const byName = Object.fromEntries(results.map((r) => [r.name, r]));
327 expect(byName["doctor:synthetic-monitor"]!.status).toBe("red");
328 expect(byName["doctor:journeys"]!.status).toBe("green");
329 expect(byName["doctor:surface-monitor"]!.status).toBe("red");
330 expect(byName["doctor:autopilot-tick"]!.status).toBe("green");
331 });
332
333 it("a stale autopilot tick is red", async () => {
334 const results = await spineDoctorChecks({
335 now: () => nowMs,
336 lastTickFinishedAt: () =>
337 new Date(nowMs - 16 * 60_000).toISOString(),
338 latest: async () => latestWith([{ name: "healthz", ageMs: 60_000 }]),
339 });
340 const tick = results.find((r) => r.name === "doctor:autopilot-tick")!;
341 expect(tick.status).toBe("red");
342 });
343
344 it("no completed tick yet (first tick after boot) is yellow, not red", async () => {
345 const results = await spineDoctorChecks({
346 now: () => nowMs,
347 lastTickFinishedAt: () => null,
348 latest: async () => latestWith([{ name: "healthz", ageMs: 60_000 }]),
349 });
350 const tick = results.find((r) => r.name === "doctor:autopilot-tick")!;
351 expect(tick.status).toBe("yellow");
352 });
353
354 it("a watcher with no rows at all is red — never has never run", async () => {
355 const results = await spineDoctorChecks({
356 now: () => nowMs,
357 lastTickFinishedAt: () => new Date(nowMs).toISOString(),
358 latest: async () => latestWith([{ name: "healthz", ageMs: 60_000 }]),
359 });
360 const surface = results.find((r) => r.name === "doctor:surface-monitor")!;
361 expect(surface.status).toBe("red");
362 expect(surface.error).toContain("never run");
363 });
364
365 it("doctor rows never count as evidence that the monitors ran", async () => {
366 const results = await spineDoctorChecks({
367 now: () => nowMs,
368 lastTickFinishedAt: () => new Date(nowMs).toISOString(),
369 latest: async () =>
370 latestWith([
371 // Only the doctor's own output is fresh — every real monitor
372 // stopped 2h ago. If the doctor fed on itself, this would
373 // read green forever.
374 { name: "doctor:synthetic-monitor", ageMs: 60_000 },
375 { name: "healthz", ageMs: 2 * 60 * 60_000 },
376 ]),
377 });
378 const core = results.find((r) => r.name === "doctor:synthetic-monitor")!;
379 expect(core.status).toBe("red");
380 });
381});
382
383describe("registry hygiene", () => {
384 it("every registry action comes from the closed allowlist", () => {
385 for (const rule of REPAIR_REGISTRY) {
386 expect(MEDIC_ACTION_NAMES).toContain(rule.action);
387 }
388 });
389
390 it("clear-failed-deploy-marker is not, and must never be, an action", () => {
391 // Deploy state belongs to the deploy scripts (see medic.ts header and
392 // the 2026-08-19 incident). This pin makes re-adding it a conscious act.
393 expect(MEDIC_ACTION_NAMES as readonly string[]).not.toContain(
394 "clear-failed-deploy-marker"
395 );
396 for (const rule of REPAIR_REGISTRY) {
397 expect(rule.action as string).not.toBe("clear-failed-deploy-marker");
398 }
399 });
400
401 it("active rules carry a positive cooldown and budget", () => {
402 for (const rule of REPAIR_REGISTRY) {
403 if (rule.action === "none") continue;
404 expect(rule.cooldownMs).toBeGreaterThan(0);
405 expect(rule.maxPerDay).toBeGreaterThanOrEqual(1);
406 }
407 });
408
409 it("every rule explains itself", () => {
410 for (const rule of REPAIR_REGISTRY) {
411 expect(rule.note.length).toBeGreaterThan(10);
412 }
413 });
414
415 it("first match wins and known names resolve as intended", () => {
416 expect(ruleFor("peer:watch")!.action).toBe("regenerate-check");
417 expect(ruleFor("dep:anthropic")!.action).toBe("regenerate-check");
418 expect(ruleFor("dep:postgres")!.action).toBe("none");
419 expect(ruleFor("healthz")!.action).toBe("none");
420 expect(ruleFor("journey:login-form")!.action).toBe("none");
421 expect(ruleFor("doctor:autopilot-tick")!.action).toBe("none");
422 expect(ruleFor("surface:/ccantynz/Gluecron.com/actions")!.action).toBe(
423 "requeue-stuck-runs"
424 );
425 // Unmapped checks have NO rule — the recorded no-rule refusal path.
426 expect(ruleFor("landing")).toBeNull();
427 expect(ruleFor("surface:/admin/users")).toBeNull();
428 });
429
430 it("every doctor check name has a declared max age", () => {
431 for (const name of [
432 "doctor:autopilot-tick",
433 "doctor:synthetic-monitor",
434 "doctor:journeys",
435 "doctor:surface-monitor",
436 ]) {
437 expect(DOCTOR_MAX_AGES[name]).toBeGreaterThan(0);
438 }
439 });
440});
Modifiedsrc/app.tsx+5−0View fileUnifiedSplit
124124import adminSelfcheckRoutes from "./routes/admin-selfcheck";
125125import adminSpineRoutes from "./routes/admin-spine";
126126import adminTruthRoutes from "./routes/admin-truth";
127import adminMedicRoutes from "./routes/admin-medic";
127128import adminErrorRoutes from "./routes/admin-errors";
128129import adminOrgRoutes from "./routes/admin-orgs";
129130import adminCommandRoutes from "./routes/admin-command";
930931// pages it summarises rather than replacing them.
931932app.route("/", adminSpineRoutes);
932933app.route("/", adminTruthRoutes);
934// /admin/medic — the medic registry, its outcome ledger, and the spine
935// doctor (watcher heartbeats). Read-only; repairs run from the autopilot
936// tick (src/lib/medic.ts, migration 0129).
937app.route("/", adminMedicRoutes);
933938app.route("/", adminErrorRoutes);
934939app.route("/", adminOrgRoutes);
935940// Unified admin command center — /admin/command
Modifiedsrc/db/schema.ts+41−0View fileUnifiedSplit
51445144
51455145export type TruthEvaluationRow = typeof truthEvaluations.$inferSelect;
51465146export type NewTruthEvaluationRow = typeof truthEvaluations.$inferInsert;
5147// Medic outcomes (2026-08-26, migration 0129) — Mission Control §6.
5148//
5149// The medic's ledger: one row per decision about a red check. outcome is
5150// 'repaired' (the red check re-ran green — the ONLY thing that earns the
5151// word), 'attempted-still-red' (the action ran, the re-verify did not come
5152// back green), or 'refused' (no rule / cooldown / budget / action-unsafe /
5153// medic-error, in `reason`). reverifiedGreen is the contract bit: a restart
5154// exiting 0 proves nothing, so repairs are only "recovered" when this is
5155// true.
5156//
5157// Cooldown + per-day budget guards in src/lib/medic.ts read this table,
5158// not memory — the deploy timer restarts the process every time main
5159// moves, and an in-memory budget would reset with it (same argument as
5160// remediation.ts counting attempts from audit_log).
5161// ---------------------------------------------------------------------------
5162
5163export const medicOutcomes = pgTable(
5164 "medic_outcomes",
5165 {
5166 id: uuid("id").primaryKey().defaultRandom(),
5167 checkName: text("check_name").notNull(),
5168 action: text("action").notNull(),
5169 outcome: text("outcome").notNull(),
5170 reason: text("reason"),
5171 detail: text("detail"),
5172 attemptedAt: timestamp("attempted_at", { withTimezone: true })
5173 .defaultNow()
5174 .notNull(),
5175 reverifiedGreen: boolean("reverified_green").default(false).notNull(),
5176 },
5177 (table) => [
5178 // The guards query "rows for this check since T" on every red tick.
5179 index("medic_outcomes_check_attempted_idx").on(
5180 table.checkName,
5181 table.attemptedAt
5182 ),
5183 ]
5184);
5185
5186export type MedicOutcome = typeof medicOutcomes.$inferSelect;
5187export type NewMedicOutcome = typeof medicOutcomes.$inferInsert;
Modifiedsrc/lib/autopilot.ts+38−0View fileUnifiedSplit
11501150 }
11511151 },
11521152 },
1153 {
1154 // Medic + spine doctor (migration 0129) — Mission Control §6.
1155 //
1156 // OPERATIONAL repair, distinct from autorepair.ts (which patches a
1157 // customer repo's CODE). A deterministic registry maps red checks to
1158 // a closed allowlist of safe actions (requeue stuck CI runs, resync
1159 // mirrors, re-run a blip-prone probe); "repaired" is only recorded
1160 // after the red check RE-RAN green, and every decline is a recorded
1161 // refusal with a reason (medic_outcomes). Also persists the
1162 // doctor:* meta-checks — is each watcher itself alive? — through the
1163 // synthetic_checks pipeline, so a dead monitor pages instead of
1164 // freezing the board on its last healthy value.
1165 //
1166 // Deliberately LAST in this list: the doctor must see the rows the
1167 // monitor tasks above wrote this tick, or a healthy spine would read
1168 // one tick stale. Cooldowns/budgets live in medic_outcomes, so the
1169 // 60s deploy timer's restarts cannot reset them.
1170 name: "medic",
1171 run: async () => {
1172 try {
1173 const { runMedicTaskOnce } = await import("./medic");
1174 const s = await runMedicTaskOnce();
1175 console.log(
1176 `[autopilot] medic: doctor green=${s.doctor.green} red=${s.doctor.red} yellow=${s.doctor.yellow} · ` +
1177 `repaired=${s.medic.repaired} attempted=${s.medic.attemptedStillRed} refused=${s.medic.refused}`
1178 );
1179 if (s.redDoctorNames.length > 0) {
1180 // Name the dead watchers — "red=2" sends an operator hunting.
1181 console.error(
1182 `[autopilot] medic: WATCHERS NOT RUNNING — ${s.redDoctorNames.join(", ")}`
1183 );
1184 }
1185 } catch (err) {
1186 console.error("[autopilot] medic: threw:", err);
1187 throw err;
1188 }
1189 },
1190 },
11531191 ];
11541192}
11551193
Addedsrc/lib/medic.ts+814−0View fileUnifiedSplit
1/**
2 * MEDIC REGISTRY + SPINE DOCTOR — Mission Control blueprint §6.
3 *
4 * OPERATIONAL repair, not code repair. `autorepair.ts` (and the AI CI
5 * healer, and the patch generator) fix a *repository's source* when a
6 * gate or scanner finds a defect. This module acts when the *running
7 * platform's* health checks go red: requeue a wedged CI queue, resync a
8 * stalled mirror fleet, re-run a blip-prone probe. Same verb — "repair" —
9 * different subject, zero shared code paths. Do not conflate them.
10 *
11 * THE VAPRON MEDIC CONTRACT (adopted verbatim):
12 *
13 * 1. Repairs are a deterministic RULEBOOK — `REPAIR_REGISTRY` maps a
14 * check name (or pattern) to exactly one action from a closed
15 * allowlist. No heuristics, no AI in the loop; the intelligence
16 * layer may PROPOSE new registry entries (blueprint §10), humans
17 * adopt them by editing this file.
18 *
19 * 2. A repair is "recovered" ONLY when the red check RE-RAN GREEN. A
20 * restart exiting 0, a sync reporting counts, an action that merely
21 * completed — none of that is recovery. `runMedic` re-runs the
22 * original check after every action and only the outcome
23 * `repaired` (with `reverified_green = true`) may claim success.
24 *
25 * 3. Refusals are RECORDED OUTCOMES with machine-readable reasons
26 * (no-rule / cooldown / budget / action-unsafe / medic-error). A
27 * medic that quietly declines looks identical to a broken medic;
28 * the `medic_outcomes` ledger is what tells them apart.
29 *
30 * WHY `clear-failed-deploy-marker` IS NOT AN ACTION. The FAILED_MARKER
31 * is how scripts/auto-update.sh parks a box on last-good after a rolled
32 * back deploy. Clearing it from inside the app would re-arm deploys over
33 * a failure no human has looked at — and deploy semantics live in the
34 * deploy scripts and their drills, not in-process. The 2026-08-19
35 * incident (an in-process-adjacent change to the deploy path stranded
36 * deploys for 5 hours) is the standing proof of what a casual hand on
37 * deploy state costs. A red deploy maps to `none`: page a human.
38 *
39 * Relationship to `remediation.ts`: the remediation ladder runs inline in
40 * each monitor task against that tick's results and already handles the
41 * verify→repair→escalate flow for its single registered action. The medic
42 * is the registry-driven generalisation the blueprint asks for — it runs
43 * from its own autopilot task over the latest red state, owns its own
44 * durable ledger (`medic_outcomes`, migration 0129), and adds the
45 * spine-doctor meta-checks. It deliberately reuses remediation's
46 * `defaultVerify` for re-running a check by name rather than reimplement
47 * the three monitors' dispatch.
48 *
49 * THE SPINE DOCTOR watches the watchers. Every monitor on /admin/spine
50 * runs from the autopilot tick; if the tick dies, every pill on the board
51 * freezes at its last value and the board renders "healthy" forever.
52 * `spineDoctorChecks()` computes, from data already persisted, whether
53 * each watcher itself ran recently — autopilot tick age, newest core
54 * synthetic row age, newest journey row age, newest surface row age — and
55 * persists the answers through the SAME `synthetic_checks` pipeline under
56 * `doctor:*` names, so they render on /admin/spine and stream over SSE
57 * with no new plumbing. Transition paging reuses `fanOutSpineAlert`
58 * (webhook-first, so a dead DB still pages). Public /status filters
59 * `doctor:*` out of its incident list — internal watcher heartbeats are
60 * not customer outages.
61 */
62
63import { and, desc, eq, gte } from "drizzle-orm";
64import { db } from "../db";
65import { medicOutcomes } from "../db/schema";
66import type { SyntheticCheckResult } from "./synthetic-monitor";
67
68// ---------------------------------------------------------------------------
69// The action allowlist
70// ---------------------------------------------------------------------------
71
72/**
73 * Closed allowlist. An action earns its place by being idempotent,
74 * in-process, and safe against a system whose state we do not fully know
75 * (it may fire against a box that is already healthy). Everything else —
76 * container restarts, deploy markers, worker flags — is somebody else's
77 * jurisdiction (autoheal, the deploy scripts, worker-stall.ts) and maps
78 * to `none`.
79 */
80export const MEDIC_ACTION_NAMES = [
81 // Finalise workflow runs abandoned in 'running' >1h. Imported from
82 // workflow-runner (its own conservatism: only rows an order of
83 // magnitude past the step timeout). Safe blind: a healthy queue has
84 // nothing to reap.
85 "requeue-stuck-runs",
86 // syncAllDue() from mirrors.ts — only syncs mirrors that are due, so
87 // running it against a healthy fleet is a no-op.
88 "resync-mirrors",
89 // Re-run the specific failing check once, to distinguish a blip from a
90 // real outage. The "repair" is discovering there was nothing to repair;
91 // the re-verify then proves it (or doesn't).
92 "regenerate-check",
93 // The explicit honest entry: this class of failure has NO safe
94 // automated repair — record the refusal and let paging do its job.
95 // Distinct from having no rule at all: `none` is a decision, absence
96 // is a coverage gap.
97 "none",
98] as const;
99
100export type MedicActionName = (typeof MEDIC_ACTION_NAMES)[number];
101
102// ---------------------------------------------------------------------------
103// The registry
104// ---------------------------------------------------------------------------
105
106export interface MedicRule {
107 /** Exact check name, or a pattern over check names. First match wins. */
108 check: string | RegExp;
109 action: MedicActionName;
110 /** Minimum quiet time between attempts for one check. */
111 cooldownMs: number;
112 /** Attempt budget per check per rolling 24h. */
113 maxPerDay: number;
114 /** Why this mapping is safe (or why it is `none`). Shown on /admin/medic. */
115 note: string;
116}
117
118export const REPAIR_REGISTRY: readonly MedicRule[] = [
119 {
120 // surface:/{owner}/{repo}/actions and .../pipeline — the pages that
121 // render the workflow queue. A queue wedged behind rows abandoned in
122 // 'running' is exactly what reapStuckRuns clears, and it only touches
123 // rows stuck longer than an hour.
124 check: /^surface:\/[^/]+\/[^/]+\/(actions|pipeline)$/,
125 action: "requeue-stuck-runs",
126 cooldownMs: 30 * 60_000,
127 maxPerDay: 6,
128 note: "Reap workflow runs abandoned in 'running' >1h; safe blind — a healthy queue has nothing to reap.",
129 },
130 {
131 // Any check whose name mentions mirrors (today: the surface walk's
132 // mirror admin page; tomorrow: a dedicated mirror-freshness check,
133 // which inherits this rule by name without an edit here).
134 check: /mirror/i,
135 action: "resync-mirrors",
136 cooldownMs: 60 * 60_000,
137 maxPerDay: 4,
138 note: "syncAllDue() syncs only mirrors that are due; idempotent against a healthy fleet.",
139 },
140 {
141 // The peer probe crosses the public internet to someone else's box —
142 // the single most blip-prone check we run.
143 check: "peer:watch",
144 action: "regenerate-check",
145 cooldownMs: 10 * 60_000,
146 maxPerDay: 24,
147 note: "Re-probe once to split network blip from real peer outage before anyone gets paged twice.",
148 },
149 {
150 // Upstream HTTP probes — other people's APIs have other people's
151 // hiccups. Re-running is the only safe move we own; a persistent red
152 // here is *their* outage or *our* revoked key, both human matters.
153 check: /^dep:(anthropic|stripe|resend)$/,
154 action: "regenerate-check",
155 cooldownMs: 10 * 60_000,
156 maxPerDay: 24,
157 note: "Re-probe the upstream once; a repeatably-red upstream is an account/key problem no automation should touch.",
158 },
159 {
160 check: /^dep:(postgres|disk|tls)$/,
161 action: "none",
162 cooldownMs: 60 * 60_000,
163 maxPerDay: 24,
164 note: "Database, disk and TLS have no safe in-process repair — a medic that touches them can only make the incident bigger. Page.",
165 },
166 {
167 check: /^(healthz|readyz)$/,
168 action: "none",
169 cooldownMs: 60 * 60_000,
170 maxPerDay: 24,
171 note: "A process cannot safely restart itself over its own liveness probe; container restarts belong to autoheal. Page.",
172 },
173 {
174 check: /^journey:/,
175 action: "none",
176 cooldownMs: 60 * 60_000,
177 maxPerDay: 24,
178 note: "Journeys prove product logic; a red one means a code defect, and code repair is autorepair/CI-healer territory. Page.",
179 },
180 {
181 check: /^doctor:/,
182 action: "none",
183 cooldownMs: 60 * 60_000,
184 maxPerDay: 24,
185 note: "A dead watcher cannot be kicked awake from inside the process it watches — that is the off-box watcher's job. Page.",
186 },
187];
188
189/** First matching rule, or null — null is the recorded `no-rule` refusal. */
190export function ruleFor(checkName: string): MedicRule | null {
191 for (const rule of REPAIR_REGISTRY) {
192 if (typeof rule.check === "string") {
193 if (rule.check === checkName) return rule;
194 } else if (rule.check.test(checkName)) {
195 return rule;
196 }
197 }
198 return null;
199}
200
201// ---------------------------------------------------------------------------
202// Outcomes
203// ---------------------------------------------------------------------------
204
205export type MedicOutcomeKind = "repaired" | "attempted-still-red" | "refused";
206
207export type MedicRefusalReason =
208 | "no-rule" // no registry entry matches this check
209 | "cooldown" // an attempt ran too recently
210 | "budget" // daily attempt budget spent (or ledger unreadable — assume spent)
211 | "action-unsafe" // the rule is `none`: this class has no safe automated repair
212 | "medic-error"; // the medic itself failed before it could act (belt-and-braces)
213
214export interface MedicResult {
215 check: string;
216 action: MedicActionName | null;
217 outcome: MedicOutcomeKind;
218 reason: MedicRefusalReason | null;
219 detail: string;
220 /** True ONLY when the original check re-ran green after the action. */
221 reverifiedGreen: boolean;
222}
223
224export interface MedicSummary {
225 considered: number;
226 repaired: number;
227 attemptedStillRed: number;
228 refused: number;
229 results: MedicResult[];
230}
231
232/** Shape the guards need from the ledger. */
233export interface MedicHistoryRow {
234 outcome: string;
235 attemptedAt: Date;
236 reason: string | null;
237}
238
239export interface MedicDeps {
240 now?: () => number;
241 rules?: readonly MedicRule[];
242 /** Re-run one check by name; null = cannot re-run. DI for tests. */
243 reverify?: (checkName: string) => Promise<SyntheticCheckResult | null>;
244 /** Execute an allowlisted action; returns a human detail. Throws on failure. */
245 execute?: (action: MedicActionName, checkName: string) => Promise<string>;
246 /** Ledger rows for a check in the last 24h, newest first. */
247 history?: (checkName: string) => Promise<MedicHistoryRow[]>;
248 /** Persist one outcome row. */
249 persist?: (row: {
250 checkName: string;
251 action: string;
252 outcome: MedicOutcomeKind;
253 reason: MedicRefusalReason | null;
254 detail: string;
255 reverifiedGreen: boolean;
256 }) => Promise<void>;
257}
258
259/** Budget window — attempts are counted per check per rolling day. */
260export const MEDIC_BUDGET_WINDOW_MS = 24 * 60 * 60 * 1000;
261
262/**
263 * A refusal identical to one already recorded within this window is not
264 * re-written. The contract is that refusals are recorded, not that they
265 * are recorded 288 times a day: a check that stays red re-enters the
266 * medic every 5-minute tick, and one `no-rule` row per hour says exactly
267 * as much as twelve while keeping the ledger readable (the mirror-sync
268 * zero-mirrors warning learned this the loud way).
269 */
270export const REFUSAL_REPEAT_WINDOW_MS = 60 * 60 * 1000;
271
272async function defaultHistory(checkName: string): Promise<MedicHistoryRow[]> {
273 const since = new Date(Date.now() - MEDIC_BUDGET_WINDOW_MS);
274 const rows = await db
275 .select({
276 outcome: medicOutcomes.outcome,
277 attemptedAt: medicOutcomes.attemptedAt,
278 reason: medicOutcomes.reason,
279 })
280 .from(medicOutcomes)
281 .where(
282 and(
283 gte(medicOutcomes.attemptedAt, since),
284 eq(medicOutcomes.checkName, checkName)
285 )
286 )
287 .orderBy(desc(medicOutcomes.attemptedAt))
288 .limit(200);
289 return rows;
290}
291
292async function defaultPersist(row: {
293 checkName: string;
294 action: string;
295 outcome: MedicOutcomeKind;
296 reason: MedicRefusalReason | null;
297 detail: string;
298 reverifiedGreen: boolean;
299}): Promise<void> {
300 await db.insert(medicOutcomes).values({
301 checkName: row.checkName,
302 action: row.action,
303 outcome: row.outcome,
304 reason: row.reason,
305 detail: row.detail.slice(0, 2000),
306 reverifiedGreen: row.reverifiedGreen,
307 });
308}
309
310/**
311 * Default action executor. Everything is imported, never reimplemented —
312 * the helpers already carry their own conservatism (reapStuckRuns only
313 * touches rows stuck >1h; syncAllDue only syncs what is due).
314 */
315export async function executeMedicAction(
316 action: MedicActionName,
317 checkName: string
318): Promise<string> {
319 switch (action) {
320 case "requeue-stuck-runs": {
321 const { reapStuckRuns } = await import("./workflow-runner");
322 const reaped = await reapStuckRuns();
323 return `reaped ${reaped} stuck workflow run(s)`;
324 }
325 case "resync-mirrors": {
326 const { syncAllDue } = await import("./mirrors");
327 const s = await syncAllDue();
328 return `mirror sync: total=${s.total} ok=${s.ok} failed=${s.failed}`;
329 }
330 case "regenerate-check": {
331 // remediation.defaultVerify already knows how to re-run any check
332 // by name across all three monitors — reuse it, don't fork it.
333 const { defaultVerify } = await import("./remediation");
334 const r = await defaultVerify(checkName);
335 return r
336 ? `re-ran ${checkName}: ${r.status}${r.error ? ` (${r.error})` : ""}`
337 : `re-ran ${checkName}: monitor could not re-run it`;
338 }
339 case "none":
340 // Unreachable by construction — runMedic refuses `none` rules
341 // before the executor. Throwing keeps that true if it ever isn't.
342 throw new Error("'none' has no executor — refuse before this point");
343 }
344}
345
346async function defaultReverify(
347 checkName: string
348): Promise<SyntheticCheckResult | null> {
349 const { defaultVerify } = await import("./remediation");
350 return defaultVerify(checkName);
351}
352
353/**
354 * The medic. For each red check: find the rule, pass the cooldown and
355 * budget guards, execute the action, RE-RUN the original check, and
356 * record what actually happened. NEVER throws; every outcome persists
357 * (identical refusals throttled to one per hour — see
358 * REFUSAL_REPEAT_WINDOW_MS).
359 */
360export async function runMedic(
361 redChecks: readonly SyntheticCheckResult[],
362 deps: MedicDeps = {}
363): Promise<MedicSummary> {
364 const now = deps.now ?? Date.now;
365 const rules = deps.rules ?? REPAIR_REGISTRY;
366 const reverify = deps.reverify ?? defaultReverify;
367 const execute = deps.execute ?? executeMedicAction;
368 const history = deps.history ?? defaultHistory;
369 const persist = deps.persist ?? defaultPersist;
370
371 const results: MedicResult[] = [];
372
373 // Sequential on purpose — these are repairs against shared state, and
374 // interleaving them reintroduces the hazard the allowlist exists to
375 // avoid (remediation.ts makes the same call).
376 for (const failing of redChecks) {
377 if (failing.status !== "red") continue;
378 const check = failing.name;
379
380 let result: MedicResult;
381 let skipPersist = false;
382 try {
383 const rule =
384 rules === REPAIR_REGISTRY
385 ? ruleFor(check)
386 : (rules.find((r) =>
387 typeof r.check === "string"
388 ? r.check === check
389 : r.check.test(check)
390 ) ?? null);
391
392 let rows: MedicHistoryRow[] = [];
393 let ledgerReadable = true;
394 try {
395 rows = await history(check);
396 } catch (err) {
397 // An unreadable ledger means we might already have burned the
398 // budget — assume the worst, never the best.
399 console.error("[medic] history read failed for", check, err);
400 ledgerReadable = false;
401 }
402
403 if (!rule) {
404 result = {
405 check,
406 action: null,
407 outcome: "refused",
408 reason: "no-rule",
409 detail:
410 "no registry entry matches this check — page instead; add a rule only if a safe deterministic repair exists",
411 reverifiedGreen: false,
412 };
413 } else if (rule.action === "none") {
414 result = {
415 check,
416 action: "none",
417 outcome: "refused",
418 reason: "action-unsafe",
419 detail: `no safe automated repair for this class — ${rule.note}`,
420 reverifiedGreen: false,
421 };
422 } else if (!ledgerReadable) {
423 result = {
424 check,
425 action: rule.action,
426 outcome: "refused",
427 reason: "budget",
428 detail: "outcome ledger unreadable — assuming budget spent",
429 reverifiedGreen: false,
430 };
431 } else {
432 const attempts = rows.filter((r) => r.outcome !== "refused");
433 const newestAttempt = attempts[0] ?? null;
434 const budgetSpent = attempts.length >= rule.maxPerDay;
435 const inCooldown =
436 newestAttempt !== null &&
437 now() - newestAttempt.attemptedAt.getTime() < rule.cooldownMs;
438
439 if (inCooldown) {
440 result = {
441 check,
442 action: rule.action,
443 outcome: "refused",
444 reason: "cooldown",
445 detail: `last attempt ${Math.round(
446 (now() - (newestAttempt as MedicHistoryRow).attemptedAt.getTime()) /
447 1000
448 )}s ago; cooldown ${Math.round(rule.cooldownMs / 1000)}s`,
449 reverifiedGreen: false,
450 };
451 } else if (budgetSpent) {
452 result = {
453 check,
454 action: rule.action,
455 outcome: "refused",
456 reason: "budget",
457 detail: `${attempts.length}/${rule.maxPerDay} attempts in 24h — budget spent, escalation is the only move left`,
458 reverifiedGreen: false,
459 };
460 } else {
461 // ── Execute ──
462 let actionDetail: string;
463 let actionThrew = false;
464 try {
465 actionDetail = await execute(rule.action, check);
466 } catch (err) {
467 actionThrew = true;
468 actionDetail = `action threw: ${
469 err instanceof Error ? err.message : String(err)
470 }`;
471 }
472
473 // ── Re-verify: the ONLY path to "repaired" ──
474 let after: SyntheticCheckResult | null = null;
475 if (!actionThrew) {
476 try {
477 after = await reverify(check);
478 } catch (err) {
479 console.error("[medic] re-verify threw for", check, err);
480 after = null;
481 }
482 }
483
484 if (after && after.status === "green") {
485 result = {
486 check,
487 action: rule.action,
488 outcome: "repaired",
489 reason: null,
490 detail: `${actionDetail}; re-verified green`,
491 reverifiedGreen: true,
492 };
493 } else {
494 // Yellow is not green. Unverifiable is not green. An action
495 // that "ran fine" earns nothing without the re-run.
496 const verdict = after
497 ? `re-verify came back ${after.status}`
498 : "re-verify unavailable — a repair that cannot be re-verified is not recovered";
499 result = {
500 check,
501 action: rule.action,
502 outcome: "attempted-still-red",
503 reason: null,
504 detail: `${actionDetail}; ${verdict}`,
505 reverifiedGreen: false,
506 };
507 }
508 }
509 }
510
511 // Throttle identical repeat refusals (same check + reason inside
512 // the window). Attempts always persist.
513 if (result.outcome === "refused" && ledgerReadable) {
514 const dup = rows.find(
515 (r) =>
516 r.outcome === "refused" &&
517 r.reason === result.reason &&
518 now() - r.attemptedAt.getTime() < REFUSAL_REPEAT_WINDOW_MS
519 );
520 if (dup) skipPersist = true;
521 }
522 } catch (err) {
523 // Belt-and-braces: the branches above already wrap every dep call,
524 // so this only fires on a medic bug — say so honestly rather than
525 // dressing it up as a repair verdict.
526 console.error("[medic] internal failure for", check, err);
527 result = {
528 check,
529 action: null,
530 outcome: "refused",
531 reason: "medic-error",
532 detail: `medic internal error: ${
533 err instanceof Error ? err.message : String(err)
534 }`,
535 reverifiedGreen: false,
536 };
537 }
538
539 if (!skipPersist) {
540 try {
541 await persist({
542 checkName: result.check,
543 action: result.action ?? "none",
544 outcome: result.outcome,
545 reason: result.reason,
546 detail: result.detail,
547 reverifiedGreen: result.reverifiedGreen,
548 });
549 } catch (err) {
550 console.error("[medic] persist failed for", check, err);
551 }
552 }
553 results.push(result);
554 }
555
556 return {
557 considered: results.length,
558 repaired: results.filter((r) => r.outcome === "repaired").length,
559 attemptedStillRed: results.filter(
560 (r) => r.outcome === "attempted-still-red"
561 ).length,
562 refused: results.filter((r) => r.outcome === "refused").length,
563 results,
564 };
565}
566
567// ---------------------------------------------------------------------------
568// Spine doctor — the watcher that watches the watchers
569// ---------------------------------------------------------------------------
570
571export const DOCTOR_CHECK_PREFIX = "doctor:";
572
573/**
574 * Max acceptable age per watcher, ~3× its cadence: the tick and the two
575 * suites that ride it run every 5 minutes (60s in .env.example, but the
576 * threshold must hold for the documented default), the surface walk every
577 * 20. One missed beat is a deploy restart; three is a dead watcher.
578 */
579export const DOCTOR_MAX_AGES: Readonly<Record<string, number>> = {
580 "doctor:autopilot-tick": 15 * 60_000,
581 "doctor:synthetic-monitor": 15 * 60_000,
582 "doctor:journeys": 15 * 60_000,
583 "doctor:surface-monitor": 60 * 60_000,
584};
585
586export interface SpineDoctorDeps {
587 now?: () => number;
588 /** ISO finishedAt of the last completed autopilot tick, or null. */
589 lastTickFinishedAt?: () => string | null;
590 /** Latest row per check (the latestStatusByCheck shape). */
591 latest?: () => Promise<Record<string, { name: string; checkedAt: Date }>>;
592}
593
594function ageLabel(ms: number): string {
595 const secs = Math.round(ms / 1000);
596 if (secs < 60) return `${secs}s`;
597 if (secs < 3600) return `${Math.round(secs / 60)}m`;
598 return `${Math.round(secs / 36) / 100}h`;
599}
600
601function doctorResult(
602 name: string,
603 newestMs: number | null,
604 nowMs: number
605): SyntheticCheckResult {
606 const maxAge = DOCTOR_MAX_AGES[name] ?? 15 * 60_000;
607 if (newestMs === null) {
608 return {
609 name,
610 status: "red",
611 durationMs: 0,
612 // The age lives in `error` because that is the column every
613 // renderer of synthetic_checks already shows as "Detail".
614 error: `no rows recorded — this watcher has never run (max age ${ageLabel(maxAge)})`,
615 };
616 }
617 const age = Math.max(0, nowMs - newestMs);
618 const detail = `newest row ${ageLabel(age)} old (max ${ageLabel(maxAge)})`;
619 return {
620 name,
621 status: age > maxAge ? "red" : "green",
622 durationMs: 0,
623 error: detail,
624 };
625}
626
627/**
628 * Compute the four meta-checks from data already persisted — no probes,
629 * no network. Pure given injected deps, which is what the age-math tests
630 * pin.
631 */
632export async function spineDoctorChecks(
633 deps: SpineDoctorDeps = {}
634): Promise<SyntheticCheckResult[]> {
635 const now = deps.now ?? Date.now;
636 const nowMs = now();
637
638 const lastTickIso = deps.lastTickFinishedAt
639 ? deps.lastTickFinishedAt()
640 : (await import("./autopilot")).getLastTick()?.finishedAt ?? null;
641
642 let latest: Record<string, { name: string; checkedAt: Date }> = {};
643 try {
644 if (deps.latest) {
645 latest = await deps.latest();
646 } else {
647 const { latestStatusByCheck } = await import("./synthetic-monitor");
648 latest = await latestStatusByCheck();
649 }
650 } catch (err) {
651 console.error("[medic] spine doctor could not read latest rows:", err);
652 latest = {};
653 }
654
655 // Newest checkedAt per watcher's namespace. Core = everything that is
656 // not another monitor's prefix and not our own output — the doctor must
657 // never count its own rows as proof the monitors ran.
658 let core: number | null = null;
659 let journey: number | null = null;
660 let surface: number | null = null;
661 for (const row of Object.values(latest)) {
662 const t = row.checkedAt?.getTime?.();
663 if (!Number.isFinite(t)) continue;
664 if (row.name.startsWith(DOCTOR_CHECK_PREFIX)) continue;
665 if (row.name.startsWith("journey:")) {
666 journey = journey === null ? t : Math.max(journey, t);
667 } else if (row.name.startsWith("surface:")) {
668 surface = surface === null ? t : Math.max(surface, t);
669 } else if (!row.name.startsWith("dep:")) {
670 core = core === null ? t : Math.max(core, t);
671 }
672 }
673
674 const results: SyntheticCheckResult[] = [];
675
676 // Autopilot tick age. Null is what the FIRST tick after boot sees
677 // (getLastTick returns completed ticks only) and the deploy timer
678 // reboots this process every time main moves — so null is yellow, not
679 // red: paging on every deploy would teach operators to ignore the one
680 // page that matters.
681 if (lastTickIso === null) {
682 results.push({
683 name: "doctor:autopilot-tick",
684 status: "yellow",
685 durationMs: 0,
686 error: "no completed tick yet in this process (first tick after boot)",
687 });
688 } else {
689 const t = new Date(lastTickIso).getTime();
690 results.push(
691 doctorResult(
692 "doctor:autopilot-tick",
693 Number.isFinite(t) ? t : null,
694 nowMs
695 )
696 );
697 }
698
699 results.push(doctorResult("doctor:synthetic-monitor", core, nowMs));
700 results.push(doctorResult("doctor:journeys", journey, nowMs));
701 results.push(doctorResult("doctor:surface-monitor", surface, nowMs));
702 return results;
703}
704
705// ---------------------------------------------------------------------------
706// The autopilot task body
707// ---------------------------------------------------------------------------
708
709/**
710 * Red rows older than this are history, not an outage in progress —
711 * comfortably above the slowest monitor cadence (20 min), so a current
712 * red is always inside the window while a red from a check that stopped
713 * existing last week is not.
714 */
715export const MEDIC_STALE_RED_MS = 30 * 60_000;
716
717export interface MedicTaskSummary {
718 doctor: { green: number; red: number; yellow: number };
719 redDoctorNames: string[];
720 medic: MedicSummary;
721}
722
723/**
724 * One iteration, run from the autopilot tick (after the monitors, so the
725 * doctor sees this tick's rows):
726 *
727 * 1. Spine doctor: compute doctor:* results, persist them through
728 * persistChecks (same table, same SSE topic — /admin/spine renders
729 * them with zero new code), and page green→red / red→green edges
730 * through the existing fanOutSpineAlert.
731 * 2. Medic: run the registry over every currently-red, current-enough
732 * check (including a doctor red — its `none` rule records the
733 * honest refusal).
734 *
735 * Never throws on its own account; anything unexpected surfaces to the
736 * tick's per-task catch, which is the mechanism that makes failures
737 * visible on /admin (see autopilot.ts's header).
738 */
739export async function runMedicTaskOnce(): Promise<MedicTaskSummary> {
740 const { latestStatusByCheck, persistChecks } = await import(
741 "./synthetic-monitor"
742 );
743
744 // One snapshot feeds everything: doctor ages, doctor edge detection,
745 // and the medic's red list.
746 const latest = await latestStatusByCheck();
747
748 const doctorResults = await spineDoctorChecks({
749 latest: async () => latest,
750 });
751 await persistChecks(doctorResults);
752
753 // Edge-triggered paging via the existing fanout — batched, like the
754 // synthetic-monitor task, so a fully dead spine is ONE message.
755 const wentRed = doctorResults.filter(
756 (r) => r.status === "red" && latest[r.name]?.status !== "red"
757 );
758 const recovered = doctorResults.filter(
759 (r) => r.status === "green" && latest[r.name]?.status === "red"
760 );
761 try {
762 const { fanOutSpineAlert } = await import("./spine-alert-fanout");
763 if (wentRed.length > 0) {
764 await fanOutSpineAlert({
765 kind: "check_red",
766 title: `🔴 Spine doctor: ${wentRed.length} watcher${wentRed.length === 1 ? "" : "s"} not running`,
767 body: wentRed.map((r) => `- ${r.name}: ${r.error ?? "red"}`).join("\n"),
768 url: "/admin/medic",
769 });
770 }
771 if (recovered.length > 0) {
772 await fanOutSpineAlert({
773 kind: "check_recovered",
774 title: `✅ Spine doctor: ${recovered.length} watcher${recovered.length === 1 ? "" : "s"} running again`,
775 body: recovered.map((r) => r.name).join(", "),
776 url: "/admin/medic",
777 });
778 }
779 } catch (err) {
780 console.error("[medic] doctor fan-out failed:", err);
781 }
782
783 // The medic's red list: current reds from the snapshot plus any doctor
784 // red computed this tick (fresher than the snapshot by definition).
785 const nowMs = Date.now();
786 const redChecks: SyntheticCheckResult[] = [
787 ...Object.values(latest)
788 .filter(
789 (r) =>
790 r.status === "red" &&
791 !r.name.startsWith(DOCTOR_CHECK_PREFIX) &&
792 nowMs - r.checkedAt.getTime() <= MEDIC_STALE_RED_MS
793 )
794 .map(({ checkedAt: _unused, ...rest }) => {
795 void _unused;
796 return rest;
797 }),
798 ...doctorResults.filter((r) => r.status === "red"),
799 ];
800
801 const medic = await runMedic(redChecks);
802
803 return {
804 doctor: {
805 green: doctorResults.filter((r) => r.status === "green").length,
806 red: doctorResults.filter((r) => r.status === "red").length,
807 yellow: doctorResults.filter((r) => r.status === "yellow").length,
808 },
809 redDoctorNames: doctorResults
810 .filter((r) => r.status === "red")
811 .map((r) => r.name),
812 medic,
813 };
814}
Addedsrc/routes/admin-medic.tsx+415−0View fileUnifiedSplit
1/**
2 * /admin/medic — the medic's rulebook and its ledger.
3 *
4 * Three sections, all read-only (every repair runs from the autopilot
5 * tick where it is guarded, re-verified and persisted — a "fix it" button
6 * here would be a second unaudited path to the same mutations, the exact
7 * drift /admin/spine's header warns against):
8 *
9 * 1. The spine doctor — is each watcher itself alive, and how old is
10 * its newest evidence.
11 * 2. The repair registry — which checks map to which allowlisted
12 * action, with cooldown/budget, and which classes are an explicit
13 * `none` (no safe automated repair — page instead).
14 * 3. The outcome ledger — last 50 medic decisions. `repaired` is only
15 * ever shown when the red check re-ran green; refusals carry their
16 * machine-readable reason, because a medic that quietly declines
17 * looks identical to a broken medic.
18 */
19
20import { Hono } from "hono";
21import { desc } from "drizzle-orm";
22import { db } from "../db";
23import { medicOutcomes } from "../db/schema";
24import { Layout } from "../views/layout";
25import { AdminShell } from "../views/admin-shell";
26import { softAuth } from "../middleware/auth";
27import type { AuthEnv } from "../middleware/auth";
28import { isSiteAdmin } from "../lib/admin";
29import { latestStatusByCheck } from "../lib/synthetic-monitor";
30import {
31 DOCTOR_CHECK_PREFIX,
32 DOCTOR_MAX_AGES,
33 MEDIC_ACTION_NAMES,
34 REPAIR_REGISTRY,
35 type MedicRule,
36} from "../lib/medic";
37
38const medic = new Hono<AuthEnv>();
39medic.use("*", softAuth);
40
41async function gate(c: any): Promise<{ user: any } | Response> {
42 const user = c.get("user");
43 if (!user) return c.redirect("/login?next=/admin/medic");
44 if (!(await isSiteAdmin(user.id))) {
45 return c.html(
46 <Layout title="Forbidden" user={user}>
47 <div class="empty-state">
48 <h2>403 — Not a site admin</h2>
49 <p>You don't have permission to view this page.</p>
50 </div>
51 </Layout>,
52 403
53 );
54 }
55 return { user };
56}
57
58// ─── data ────────────────────────────────────────────────────────────
59
60function relTime(d: Date | string | null): string {
61 if (!d) return "—";
62 const t = new Date(d).getTime();
63 if (!Number.isFinite(t)) return "—";
64 const secs = Math.max(0, Math.round((Date.now() - t) / 1000));
65 if (secs < 60) return `${secs}s ago`;
66 if (secs < 3600) return `${Math.round(secs / 60)}m ago`;
67 if (secs < 86400) return `${Math.round(secs / 3600)}h ago`;
68 return `${Math.round(secs / 86400)}d ago`;
69}
70
71function ruleCheckLabel(rule: MedicRule): string {
72 return typeof rule.check === "string" ? rule.check : String(rule.check);
73}
74
75function cooldownLabel(ms: number): string {
76 const mins = Math.round(ms / 60_000);
77 return mins >= 60 ? `${Math.round(mins / 60)}h` : `${mins}m`;
78}
79
80async function loadOutcomes() {
81 try {
82 return await db
83 .select()
84 .from(medicOutcomes)
85 .orderBy(desc(medicOutcomes.attemptedAt))
86 .limit(50);
87 } catch (err) {
88 console.error("[admin-medic] outcomes query failed:", err);
89 return [];
90 }
91}
92
93/** Outcome → pill class. repaired=evergreen, attempted=amber, refused=neutral. */
94function outcomePill(outcome: string): string {
95 if (outcome === "repaired") return "ok";
96 if (outcome === "attempted-still-red") return "warn";
97 return "idle";
98}
99
100// ─── route ───────────────────────────────────────────────────────────
101
102medic.get("/admin/medic", async (c) => {
103 const g = await gate(c);
104 if (g instanceof Response) return g;
105 const { user } = g;
106
107 const [latest, outcomes] = await Promise.all([
108 latestStatusByCheck(),
109 loadOutcomes(),
110 ]);
111
112 const doctorRows = Object.values(latest)
113 .filter((r) => r.name.startsWith(DOCTOR_CHECK_PREFIX))
114 .sort((a, b) => a.name.localeCompare(b.name));
115 const doctorRed = doctorRows.filter((r) => r.status === "red").length;
116
117 const repaired = outcomes.filter((o) => o.outcome === "repaired").length;
118 const attempted = outcomes.filter(
119 (o) => o.outcome === "attempted-still-red"
120 ).length;
121 const refused = outcomes.filter((o) => o.outcome === "refused").length;
122
123 return c.html(
124 <AdminShell active="medic" title="Medic" user={user}>
125 <style dangerouslySetInnerHTML={{ __html: medicStyles }} />
126
127 {/* ── Headline ── */}
128 <div class={"medic-hero " + (doctorRed > 0 ? "is-bad" : "is-ok")}>
129 <div>
130 <p class="medic-hero-eyebrow">Operational repair</p>
131 <h1 class="medic-hero-title">
132 {doctorRed === 0
133 ? "All watchers running"
134 : `${doctorRed} watcher${doctorRed === 1 ? "" : "s"} not running`}
135 </h1>
136 <p class="medic-hero-sub">
137 Deterministic rulebook over red checks. A repair only counts as
138 recovered when the red check re-ran green; every decline is a
139 recorded refusal with a reason.
140 </p>
141 </div>
142 <div class="medic-hero-meta">
143 <div>
144 <span class="medic-hero-metric">{repaired}</span>
145 <span class="medic-hero-label">repaired</span>
146 </div>
147 <div>
148 <span class="medic-hero-metric">{attempted}</span>
149 <span class="medic-hero-label">attempted</span>
150 </div>
151 <div>
152 <span class="medic-hero-metric">{refused}</span>
153 <span class="medic-hero-label">refused</span>
154 </div>
155 </div>
156 </div>
157
158 {/* ── Spine doctor ── */}
159 <section class="medic-section">
160 <header class="medic-section-head">
161 <div>
162 <h2 class="medic-section-title">
163 <span
164 class={
165 "medic-dot medic-dot-" + (doctorRed > 0 ? "red" : "green")
166 }
167 aria-hidden="true"
168 />
169 Spine doctor — the watchers' heartbeats
170 </h2>
171 <p class="medic-section-blurb">
172 Every monitor on <a href="/admin/spine">/admin/spine</a> runs
173 from the autopilot tick; if the tick dies, every pill freezes at
174 its last value and the board reads healthy forever. These
175 meta-checks measure how old each watcher's newest evidence is
176 and go red at ~3× its cadence. They persist as{" "}
177 <code>doctor:*</code> rows in the same table as everything else,
178 so a dead watcher pages through the same fanout as a dead page.
179 </p>
180 </div>
181 </header>
182 {doctorRows.length === 0 ? (
183 <p class="medic-empty">
184 No doctor rows recorded yet — the medic task runs on the autopilot
185 tick; give it one tick, and if this stays empty the autopilot
186 itself is the patient.
187 </p>
188 ) : (
189 <div class="medic-table-wrap">
190 <table class="medic-table">
191 <thead>
192 <tr>
193 <th>Watcher</th>
194 <th class="medic-th-c">Status</th>
195 <th>Age detail</th>
196 <th class="medic-th-c">Max age</th>
197 <th class="medic-th-c">Checked</th>
198 </tr>
199 </thead>
200 <tbody>
201 {doctorRows.map((r) => (
202 <tr>
203 <td class="medic-name">
204 {r.name.slice(DOCTOR_CHECK_PREFIX.length)}
205 </td>
206 <td class="medic-td-c">
207 <span class={"medic-pill medic-pill-" + r.status}>
208 {r.status}
209 </span>
210 </td>
211 <td class="medic-detail">{r.error ?? "—"}</td>
212 <td class="medic-td-c medic-mono">
213 {cooldownLabel(DOCTOR_MAX_AGES[r.name] ?? 15 * 60_000)}
214 </td>
215 <td class="medic-td-c medic-muted">
216 {relTime(r.checkedAt)}
217 </td>
218 </tr>
219 ))}
220 </tbody>
221 </table>
222 </div>
223 )}
224 </section>
225
226 {/* ── Registry ── */}
227 <section class="medic-section">
228 <header class="medic-section-head">
229 <div>
230 <h2 class="medic-section-title">
231 <span class="medic-dot medic-dot-idle" aria-hidden="true" />
232 Repair registry
233 </h2>
234 <p class="medic-section-blurb">
235 The whole rulebook — first match wins, everything unmatched is a
236 recorded <code>no-rule</code> refusal. Actions come from a closed
237 allowlist ({MEDIC_ACTION_NAMES.join(", ")});{" "}
238 <code>clear-failed-deploy-marker</code> is deliberately excluded
239 because deploy state belongs to the deploy scripts, not an
240 in-process medic. Editing the rulebook means editing{" "}
241 <code>src/lib/medic.ts</code> — by design, not by accident.
242 </p>
243 </div>
244 </header>
245 <div class="medic-table-wrap">
246 <table class="medic-table">
247 <thead>
248 <tr>
249 <th>Check / pattern</th>
250 <th class="medic-th-c">Action</th>
251 <th class="medic-th-c">Cooldown</th>
252 <th class="medic-th-c">Budget/day</th>
253 <th>Why this is safe (or why it's none)</th>
254 </tr>
255 </thead>
256 <tbody>
257 {REPAIR_REGISTRY.map((rule) => (
258 <tr>
259 <td class="medic-name medic-mono">{ruleCheckLabel(rule)}</td>
260 <td class="medic-td-c">
261 <span
262 class={
263 "medic-pill " +
264 (rule.action === "none"
265 ? "medic-pill-idle"
266 : "medic-pill-action")
267 }
268 >
269 {rule.action}
270 </span>
271 </td>
272 <td class="medic-td-c medic-mono">
273 {rule.action === "none"
274 ? "—"
275 : cooldownLabel(rule.cooldownMs)}
276 </td>
277 <td class="medic-td-c medic-mono">
278 {rule.action === "none" ? "—" : rule.maxPerDay}
279 </td>
280 <td class="medic-detail">{rule.note}</td>
281 </tr>
282 ))}
283 </tbody>
284 </table>
285 </div>
286 </section>
287
288 {/* ── Outcome ledger ── */}
289 <section class="medic-section">
290 <header class="medic-section-head">
291 <div>
292 <h2 class="medic-section-title">
293 <span class="medic-dot medic-dot-idle" aria-hidden="true" />
294 Outcome ledger
295 </h2>
296 <p class="medic-section-blurb">
297 Last 50 decisions. <code>repaired</code> means the red check
298 re-ran green — nothing else earns the word.{" "}
299 <code>attempted-still-red</code> means the action ran and the
300 re-verify did not come back green. Refusals carry their reason;
301 identical repeats are throttled to one row per hour so the
302 ledger stays readable.
303 </p>
304 </div>
305 </header>
306 {outcomes.length === 0 ? (
307 <p class="medic-empty">
308 No medic decisions recorded yet — which, with a green board above,
309 means nothing has needed one.
310 </p>
311 ) : (
312 <div class="medic-table-wrap">
313 <table class="medic-table">
314 <thead>
315 <tr>
316 <th>Check</th>
317 <th class="medic-th-c">Action</th>
318 <th class="medic-th-c">Outcome</th>
319 <th class="medic-th-c">Reason</th>
320 <th class="medic-th-c">Re-verified green</th>
321 <th>Detail</th>
322 <th class="medic-th-c">When</th>
323 </tr>
324 </thead>
325 <tbody>
326 {outcomes.map((o) => (
327 <tr>
328 <td class="medic-name">{o.checkName}</td>
329 <td class="medic-td-c medic-mono">{o.action}</td>
330 <td class="medic-td-c">
331 <span
332 class={"medic-pill medic-out-" + outcomePill(o.outcome)}
333 >
334 {o.outcome}
335 </span>
336 </td>
337 <td class="medic-td-c medic-mono">{o.reason ?? "—"}</td>
338 <td class="medic-td-c">
339 {o.reverifiedGreen ? (
340 <span class="medic-pill medic-out-ok">yes</span>
341 ) : (
342 <span class="medic-muted">no</span>
343 )}
344 </td>
345 <td class="medic-detail">{o.detail ?? "—"}</td>
346 <td class="medic-td-c medic-muted">
347 {relTime(o.attemptedAt)}
348 </td>
349 </tr>
350 ))}
351 </tbody>
352 </table>
353 </div>
354 )}
355 </section>
356 </AdminShell>
357 );
358});
359
360/* Tokens only — every colour reads a token defined in BOTH theme blocks
361 (the /admin/spine styles' hard-learned rule), so nothing can fall
362 through to a dark value on the light theme. repaired=evergreen
363 (--green), attempted=amber (--yellow), refused=neutral (--text-muted). */
364const medicStyles = `
365.medic-hero{display:flex;justify-content:space-between;align-items:center;gap:24px;
366 padding:24px;border-radius:12px;margin-bottom:24px;border:1px solid var(--border);
367 background:var(--bg-elevated)}
368.medic-hero.is-bad{border-color:var(--red);background:linear-gradient(90deg,color-mix(in srgb,var(--red) 10%,transparent),transparent)}
369.medic-hero.is-ok{border-color:var(--green);background:linear-gradient(90deg,color-mix(in srgb,var(--green) 10%,transparent),transparent)}
370.medic-hero-eyebrow{margin:0 0 4px;font-size:11px;letter-spacing:.08em;text-transform:uppercase;
371 color:var(--text-muted)}
372.medic-hero-title{margin:0 0 4px;font-size:24px;line-height:1.2;color:var(--text-strong)}
373.medic-hero-sub{margin:0;color:var(--text-muted);font-size:13px;max-width:64ch}
374.medic-hero-meta{display:flex;gap:28px;text-align:right}
375.medic-hero-metric{display:block;font-size:20px;font-weight:600;color:var(--text-strong)}
376.medic-hero-label{display:block;font-size:11px;color:var(--text-muted);
377 text-transform:uppercase;letter-spacing:.05em}
378.medic-section{margin-bottom:28px;border:1px solid var(--border);border-radius:10px;
379 background:var(--bg-elevated);overflow:hidden}
380.medic-section-head{display:flex;justify-content:space-between;align-items:flex-start;gap:16px;
381 padding:16px 18px;border-bottom:1px solid var(--border)}
382.medic-section-title{margin:0 0 4px;font-size:15px;display:flex;align-items:center;gap:8px;
383 color:var(--text-strong)}
384.medic-section-blurb{margin:0;font-size:12px;color:var(--text-muted);max-width:78ch}
385.medic-dot{width:9px;height:9px;border-radius:50%;display:inline-block;flex:none}
386.medic-dot-green{background:var(--green)}.medic-dot-red{background:var(--red)}
387.medic-dot-idle{background:var(--text-muted)}
388.medic-table-wrap{overflow-x:auto}
389.medic-table{width:100%;border-collapse:collapse;font-size:13px}
390.medic-table th{text-align:left;padding:9px 14px;font-size:11px;text-transform:uppercase;
391 letter-spacing:.05em;color:var(--text-muted);border-bottom:1px solid var(--border)}
392.medic-table td{padding:9px 14px;border-bottom:1px solid var(--border-subtle);vertical-align:top}
393.medic-th-c,.medic-td-c{text-align:center}
394.medic-name{font-weight:500;word-break:break-all;color:var(--text-strong)}
395.medic-detail{color:var(--text-muted);font-size:12px;max-width:52ch}
396.medic-muted{color:var(--text-muted);font-size:12px;white-space:nowrap}
397.medic-mono{font-family:var(--font-mono,ui-monospace,SFMono-Regular,Menlo,monospace);font-size:12px}
398.medic-pill{display:inline-block;padding:2px 8px;border-radius:999px;font-size:11px;
399 font-weight:600}
400.medic-pill-green{background:color-mix(in srgb,var(--green) 15%,transparent);color:var(--green)}
401.medic-pill-yellow{background:color-mix(in srgb,var(--yellow) 15%,transparent);color:var(--yellow)}
402.medic-pill-red{background:color-mix(in srgb,var(--red) 15%,transparent);color:var(--red)}
403.medic-pill-idle{background:color-mix(in srgb,var(--text-muted) 15%,transparent);color:var(--text-muted)}
404.medic-pill-action{background:color-mix(in srgb,var(--accent) 14%,transparent);color:var(--accent)}
405.medic-out-ok{background:color-mix(in srgb,var(--green) 15%,transparent);color:var(--green)}
406.medic-out-warn{background:color-mix(in srgb,var(--yellow) 15%,transparent);color:var(--yellow)}
407.medic-out-idle{background:color-mix(in srgb,var(--text-muted) 15%,transparent);color:var(--text-muted)}
408.medic-empty{padding:18px;margin:0;color:var(--text-muted);font-size:13px}
409@media (max-width:720px){
410 .medic-hero{flex-direction:column;align-items:flex-start}
411 .medic-hero-meta{text-align:left}
412}
413`;
414
415export default medic;
Modifiedsrc/routes/status.tsx+9−1View fileUnifiedSplit
361361 // it, publishing internal admin paths to customers.
362362 let recentIncidentRows: Awaited<ReturnType<typeof recentRedChecks>> = [];
363363 try {
364 recentIncidentRows = await recentRedChecks(24, 10, { coreOnly: true });
364 // `doctor:*` rows (spine-doctor watcher heartbeats, migration 0129)
365 // share the core namespace so they ride /admin/spine for free, but a
366 // dead internal watcher is an engineering page, not a customer
367 // outage — exactly the class of leak the coreOnly comment above
368 // documents. Filtered here because coreOnly lives in the locked
369 // synthetic-monitor.ts.
370 recentIncidentRows = (
371 await recentRedChecks(24, 10, { coreOnly: true })
372 ).filter((r) => !r.name.startsWith("doctor:"));
365373 } catch {
366374 recentIncidentRows = [];
367375 }
368376
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts