feat(paging): test button + weekly drill — paging you can't test is paging you don't have #5537
7 changed files+1194−0
Addeddocs/ESCALATION-POLICY.md+55−0View fileUnifiedSplit
@@ -0,0 +1,55 @@
1# Escalation policy
2
3Plain-language, founder-scale truth. This is the whole policy — there is
4no hidden rotation behind it. Rendered verbatim at /admin/paging.
5
6## Who is paged
7
8One person: the founder (site admin). There is no on-call rotation, no
9second tier, no follow-the-sun. If the founder's phone does not receive
10the page, nobody does — which is exactly why the paging path itself is
11drilled weekly and testable on demand.
12
13## Channels, in delivery order
14
151. Webhook page (MONITOR_ALERT_WEBHOOK_URL — an ntfy/Pushover/Slack-style
16 receiver on the founder's phone). Fired FIRST and needs no database,
17 so it still works when the database is what broke.
182. In-app notification (the bell) for every site admin.
193. Email to every site admin.
20
21## Which severities page, and when
22
23- critical (a check went red, a deploy failed): pages immediately, any
24 hour, every day. These mean users may be seeing a broken site.
25- warning (error spike, drill exercising the warning path): pages at
26 normal phone priority. Look same working day.
27- info (recovery notices, manual paging tests): no action expected.
28 Recoveries are sent on purpose — an outage page with no recovery page
29 leaves the reader assuming the worst.
30
31## How we know paging works (the honest version)
32
33Pressing "Send test page now" on /admin/paging, and the automated weekly
34drill, verify that the webhook endpoint ACCEPTED the page (HTTP 2xx).
35That is not proof a phone buzzed — receivers return 200 before the device
36is reached. The human confirmation step is: after a test page, look at
37your device.
38
39## The dead-man backstop chain
40
41A broken pager cannot page about itself, so failure is layered:
42
431. If the weekly drill's page is refused or the webhook is unconfigured,
44 a `doctor:paging` red row lands in the synthetic-checks pipeline and
45 renders on /status and /admin/spine — visible the next time anyone
46 looks anywhere.
472. If the whole box (or the autopilot loop) goes silent, the external
48 healthchecks dead-man stops receiving its ping and pages independently
49 from outside the box, and the GitHub-mirror heartbeat provides an
50 off-platform pulse.
51
52If all three layers are dark at once, the box, the pager, and the
53external watcher have failed together — at founder scale the remaining
54detector is the founder noticing, and this policy says so rather than
55pretending otherwise.
Addedsrc/__tests__/paging-drill.test.ts+292−0View fileUnifiedSplit
@@ -0,0 +1,292 @@
1/**
2 * Paging drill — the pager itself is a monitored component (blueprint §7).
3 *
4 * Pins, in order of what they cost when broken:
5 * 1. Test pages are unmistakably labeled and carry the right severity —
6 * a drill that reads like a real outage trains the owner to ignore
7 * real outages.
8 * 2. An unconfigured/refused webhook is reported as sent:false AND lands
9 * as a doctor:paging RED row — a broken pager can't page about
10 * itself, so the red row is the leg that carries the news.
11 * 3. The weekly guard actually holds for 7 days and hydrates from the
12 * persisted ledger — without hydration every deploy re-pages.
13 * 4. The webhook URL (a capability: whoever holds it can page the
14 * owner) never appears in any returned or rendered string.
15 */
16
17import { describe, expect, it } from "bun:test";
18import {
19 buildTestPageAlert,
20 drillSeverityForWeek,
21 kindForSeverity,
22 sendTestPage,
23 weeklyPagingDrill,
24 __resetPagingDrillForTests,
25 PAGING_AUDIT_ACTION,
26 PAGING_CHECK_NAME,
27 PAGING_DRILL_INTERVAL_MS,
28} from "../lib/paging-drill";
29import { alertSeverity } from "../lib/spine-alert-fanout";
30import type { SyntheticCheckResult } from "../lib/synthetic-monitor";
31// Static import: the route pulls a large module graph (Layout, AdminShell)
32// and a dynamic import inside a test body eats the per-test timeout.
33import { webhookPresence } from "../routes/admin-paging";
34
35const DAY_MS = 24 * 60 * 60 * 1000;
36
37/** A URL that would be catastrophic to leak — distinctive so contains() is airtight. */
38const SECRET_URL = "https://ntfy.sh/leak-canary-topic-8d1f";
39
40function okFetch(): {
41 fetchImpl: typeof fetch;
42 calls: Array<{ url: string; body: string }>;
43} {
44 const calls: Array<{ url: string; body: string }> = [];
45 const fetchImpl = (async (url: any, init?: any) => {
46 calls.push({ url: String(url), body: String(init?.body ?? "") });
47 return new Response("ok", { status: 200 });
48 }) as typeof fetch;
49 return { fetchImpl, calls };
50}
51
52function auditSink(): {
53 auditFn: (opts: any) => Promise<void>;
54 rows: any[];
55} {
56 const rows: any[] = [];
57 return {
58 auditFn: async (opts: any) => {
59 rows.push(opts);
60 },
61 rows,
62 };
63}
64
65describe("buildTestPageAlert", () => {
66 it("manual pages are labeled, timestamped, and info severity", () => {
67 const now = new Date("2026-08-26T12:00:00.000Z");
68 const { alert, severity } = buildTestPageAlert("manual", now);
69 expect(alert.title).toBe("PAGING TEST — reply not required");
70 expect(alert.body).toContain("Manual test page");
71 expect(alert.body).toContain("2026-08-26T12:00:00.000Z");
72 expect(alert.body).toContain("No action required");
73 expect(severity).toBe("info");
74 // Severity is derived from kind by the read-only fan-out module — pin
75 // that our kind choice really maps to the severity we claim.
76 expect(alertSeverity(alert.kind)).toBe("info");
77 expect(alert.url).toBe("/admin/paging");
78 });
79
80 it("drill pages name the drill, the timestamp, and their severity", () => {
81 const now = new Date("2026-08-26T12:00:00.000Z");
82 const { alert, severity } = buildTestPageAlert("drill", now);
83 expect(alert.title).toBe("PAGING TEST — reply not required");
84 expect(alert.body).toContain("Weekly paging drill");
85 expect(alert.body).toContain("2026-08-26T12:00:00.000Z");
86 expect(alert.body).toContain(severity);
87 expect(severity).toBe(drillSeverityForWeek(now.getTime()));
88 expect(alertSeverity(alert.kind)).toBe(severity);
89 });
90
91 it("drill severity alternates week to week so both priority paths get exercised", () => {
92 const t0 = Date.parse("2026-08-26T12:00:00.000Z");
93 const a = drillSeverityForWeek(t0);
94 const b = drillSeverityForWeek(t0 + 7 * DAY_MS);
95 const c = drillSeverityForWeek(t0 + 14 * DAY_MS);
96 expect(a).not.toBe(b);
97 expect(a).toBe(c);
98 expect(["warning", "info"]).toContain(a);
99 expect(["warning", "info"]).toContain(b);
100 });
101
102 it("kindForSeverity round-trips through the fan-out module's severity map", () => {
103 expect(alertSeverity(kindForSeverity("info"))).toBe("info");
104 expect(alertSeverity(kindForSeverity("warning"))).toBe("warning");
105 });
106});
107
108describe("sendTestPage", () => {
109 it("sends through the real postAlertWebhook (DI'd fetch) and audits accepted:true", async () => {
110 const { fetchImpl, calls } = okFetch();
111 const { auditFn, rows } = auditSink();
112 const result = await sendTestPage("manual", {
113 webhook: { url: SECRET_URL, fetchImpl },
114 auditFn,
115 });
116 expect(result.sent).toBe(true);
117 expect(result.statusCode).toBe(200);
118 expect(calls).toHaveLength(1);
119 const payload = JSON.parse(calls[0]!.body);
120 expect(payload.title).toBe("PAGING TEST — reply not required");
121 expect(payload.severity).toBe("info");
122 expect(rows).toHaveLength(1);
123 expect(rows[0].action).toBe(PAGING_AUDIT_ACTION);
124 expect(rows[0].metadata.kind).toBe("manual");
125 expect(rows[0].metadata.accepted).toBe(true);
126 expect(rows[0].metadata.statusCode).toBe(200);
127 });
128
129 it("unconfigured webhook returns sent:false with a reason and audits the refusal", async () => {
130 const { auditFn, rows } = auditSink();
131 // url:"" overrides any env value — postAlertWebhook's ?? chain only
132 // falls through on nullish, so this is the unconfigured path even on a
133 // box where MONITOR_ALERT_WEBHOOK_URL is set.
134 const result = await sendTestPage("drill", {
135 webhook: { url: "" },
136 auditFn,
137 });
138 expect(result.sent).toBe(false);
139 expect(result.reason).toContain("MONITOR_ALERT_WEBHOOK_URL not set");
140 expect(rows[0].metadata.accepted).toBe(false);
141 expect(rows[0].metadata.reason).toContain("not set");
142 });
143
144 it("a refusing endpoint (HTTP 500) reports sent:false without throwing", async () => {
145 const fetchImpl = (async () =>
146 new Response("nope", { status: 500 })) as unknown as typeof fetch;
147 const { auditFn, rows } = auditSink();
148 const result = await sendTestPage("manual", {
149 webhook: { url: SECRET_URL, fetchImpl },
150 auditFn,
151 });
152 expect(result.sent).toBe(false);
153 expect(result.reason).toBe("HTTP 500");
154 expect(result.statusCode).toBe(500);
155 expect(rows[0].metadata.accepted).toBe(false);
156 });
157
158 it("an audit failure never breaks the send", async () => {
159 const { fetchImpl } = okFetch();
160 const result = await sendTestPage("manual", {
161 webhook: { url: SECRET_URL, fetchImpl },
162 auditFn: async () => {
163 throw new Error("db down");
164 },
165 });
166 expect(result.sent).toBe(true);
167 });
168});
169
170describe("weeklyPagingDrill", () => {
171 it("a refused/unconfigured webhook persists a doctor:paging RED row", async () => {
172 __resetPagingDrillForTests();
173 const persisted: SyntheticCheckResult[][] = [];
174 const s = await weeklyPagingDrill({
175 now: () => Date.parse("2026-08-26T12:00:00.000Z"),
176 loadLastDrillAt: async () => 0,
177 send: async () => ({
178 sent: false,
179 reason: "MONITOR_ALERT_WEBHOOK_URL not set",
180 }),
181 persist: async (rows) => {
182 persisted.push(rows);
183 },
184 });
185 expect(s.ran).toBe(true);
186 expect(s.accepted).toBe(false);
187 expect(persisted).toHaveLength(1);
188 const row = persisted[0]![0]!;
189 expect(row.name).toBe(PAGING_CHECK_NAME);
190 expect(row.status).toBe("red");
191 expect(row.error).toContain("NOT reaching the owner");
192 });
193
194 it("an accepted drill persists a green doctor:paging row (recovery renders)", async () => {
195 __resetPagingDrillForTests();
196 const persisted: SyntheticCheckResult[][] = [];
197 const s = await weeklyPagingDrill({
198 now: () => Date.parse("2026-08-26T12:00:00.000Z"),
199 loadLastDrillAt: async () => 0,
200 send: async () => ({ sent: true, statusCode: 200 }),
201 persist: async (rows) => {
202 persisted.push(rows);
203 },
204 });
205 expect(s.ran).toBe(true);
206 expect(s.accepted).toBe(true);
207 expect(persisted[0]![0]!.name).toBe(PAGING_CHECK_NAME);
208 expect(persisted[0]![0]!.status).toBe("green");
209 });
210
211 it("guard math: runs, then holds for 7 days, then runs again", async () => {
212 __resetPagingDrillForTests();
213 const t0 = Date.parse("2026-08-26T12:00:00.000Z");
214 let sends = 0;
215 const deps = (nowMs: number) => ({
216 now: () => nowMs,
217 loadLastDrillAt: async () => 0,
218 send: async () => {
219 sends += 1;
220 return { sent: true, statusCode: 200 };
221 },
222 persist: async () => {},
223 });
224 expect((await weeklyPagingDrill(deps(t0))).ran).toBe(true);
225 expect((await weeklyPagingDrill(deps(t0 + DAY_MS))).ran).toBe(false);
226 expect(
227 (await weeklyPagingDrill(deps(t0 + PAGING_DRILL_INTERVAL_MS - 1))).ran
228 ).toBe(false);
229 expect(
230 (await weeklyPagingDrill(deps(t0 + PAGING_DRILL_INTERVAL_MS))).ran
231 ).toBe(true);
232 expect(sends).toBe(2);
233 });
234
235 it("guard hydrates from the persisted ledger — a deploy restart must not re-page", async () => {
236 __resetPagingDrillForTests();
237 const t0 = Date.parse("2026-08-26T12:00:00.000Z");
238 let sends = 0;
239 const s = await weeklyPagingDrill({
240 now: () => t0,
241 // "Fresh process, but the audit log says a drill ran 1 day ago."
242 loadLastDrillAt: async () => t0 - DAY_MS,
243 send: async () => {
244 sends += 1;
245 return { sent: true, statusCode: 200 };
246 },
247 persist: async () => {},
248 });
249 expect(s.ran).toBe(false);
250 expect(sends).toBe(0);
251 });
252});
253
254describe("the webhook URL never leaks", () => {
255 it("no returned or audited string contains the URL", async () => {
256 const { fetchImpl } = okFetch();
257 const { auditFn, rows } = auditSink();
258 const result = await sendTestPage("manual", {
259 webhook: { url: SECRET_URL, fetchImpl },
260 auditFn,
261 });
262 expect(JSON.stringify(result)).not.toContain(SECRET_URL);
263 expect(JSON.stringify(rows)).not.toContain(SECRET_URL);
264 const { alert } = buildTestPageAlert("manual");
265 expect(JSON.stringify(alert)).not.toContain(SECRET_URL);
266 });
267
268 it("webhookPresence is boolean-only by shape — the value cannot reach a render", () => {
269 const p = webhookPresence({
270 MONITOR_ALERT_WEBHOOK_URL: SECRET_URL,
271 } as NodeJS.ProcessEnv);
272 expect(p.configured).toBe(true);
273 // Exactly one key, and it is a boolean: adding a string field to this
274 // shape is how a future edit would leak the URL — fail loudly then.
275 expect(Object.keys(p)).toEqual(["configured"]);
276 expect(
277 webhookPresence({} as NodeJS.ProcessEnv).configured
278 ).toBe(false);
279 });
280
281 it("the route source touches MONITOR_ALERT_WEBHOOK_URL exactly once, inside webhookPresence", async () => {
282 const src = await Bun.file("src/routes/admin-paging.tsx").text();
283 const uses = src.match(/process\.env\.MONITOR_ALERT_WEBHOOK_URL/g) ?? [];
284 // env-drift/copy references by NAME are fine; the VALUE must only be
285 // read in webhookPresence, which returns a boolean.
286 expect(
287 (src.match(/env\.MONITOR_ALERT_WEBHOOK_URL/g) ?? []).length
288 ).toBeLessThanOrEqual(2); // default-param `process.env` + the Boolean() read
289 expect(uses.length).toBeLessThanOrEqual(1);
290 expect(src).toContain("Boolean((env.MONITOR_ALERT_WEBHOOK_URL");
291 });
292});
Modifiedsrc/app.tsx+6−0View fileUnifiedSplit
@@ -125,6 +125,7 @@ import adminSelfcheckRoutes from "./routes/admin-selfcheck";
125125import adminSpineRoutes from "./routes/admin-spine";
126126import adminTruthRoutes from "./routes/admin-truth";
127127import adminMedicRoutes from "./routes/admin-medic";
128import adminPagingRoutes from "./routes/admin-paging";
128129import adminErrorRoutes from "./routes/admin-errors";
129130import adminOrgRoutes from "./routes/admin-orgs";
130131import adminCommandRoutes from "./routes/admin-command";
@@ -935,6 +936,11 @@ app.route("/", adminTruthRoutes);
935936// doctor (watcher heartbeats). Read-only; repairs run from the autopilot
936937// tick (src/lib/medic.ts, migration 0129).
937938app.route("/", adminMedicRoutes);
939// /admin/paging — paging test button + weekly drill ledger (blueprint §7).
940// Mounted in the pre-insightRoutes block for the same shadowing reason as
941// adminOpsRoutes: POST /admin/paging/test must not be eaten by a
942// /:owner/:repo catch-all.
943app.route("/", adminPagingRoutes);
938944app.route("/", adminErrorRoutes);
939945app.route("/", adminOrgRoutes);
940946// Unified admin command center — /admin/command
Modifiedsrc/lib/autopilot.ts+33−0View fileUnifiedSplit
@@ -97,6 +97,7 @@ import { sendSmartDigestsToAll } from "./smart-digest";
9797import { runAiLoopSweepOnce } from "./ai-loop";
9898import { runDepUpdateSweepOnce } from "./dep-updater-sweep";
9999import { runTruthLedgerTaskOnce } from "./truth-ledger";
100import { weeklyPagingDrill } from "./paging-drill";
100101
101102export interface AutopilotTaskResult {
102103 name: string;
@@ -1188,6 +1189,38 @@ export function defaultTasks(): AutopilotTask[] {
11881189 }
11891190 },
11901191 },
1192 {
1193 // Weekly paging drill — alerting's rot-check (blueprint §7). Sends a
1194 // clearly-labeled synthetic page through the real webhook path once
1195 // per 7 days (guard + severity alternation live in the lib, hydrated
1196 // from the audit log so deploys don't re-fire it). A refused or
1197 // unconfigured webhook persists a `doctor:paging` red row through the
1198 // synthetic_checks pipeline — a broken pager can't page about itself,
1199 // so the news travels by red row (/status, /admin/spine) and the
1200 // external healthchecks dead-man covers total silence.
1201 name: "paging-drill",
1202 run: async () => {
1203 try {
1204 const s = await weeklyPagingDrill();
1205 if (s.ran) {
1206 console.log(
1207 `[autopilot] paging-drill: accepted=${s.accepted}` +
1208 (s.reason ? ` reason=${s.reason}` : "")
1209 );
1210 }
1211 if (s.ran && !s.accepted) {
1212 // ok:false on /admin — the tick-level failure list is one more
1213 // surface that says the pager is broken.
1214 throw new Error(
1215 `paging drill: webhook did not accept the page (${s.reason ?? "unknown"}) — see /admin/paging`
1216 );
1217 }
1218 } catch (err) {
1219 console.error("[autopilot] paging-drill: threw:", err);
1220 throw err;
1221 }
1222 },
1223 },
11911224 ];
11921225}
11931226
Addedsrc/lib/paging-drill.ts+357−0View fileUnifiedSplit
@@ -0,0 +1,357 @@
1/**
2 * Paging drill — alerting's rot-check (Mission Control blueprint §7).
3 *
4 * The scar this heals: on 2026-08-22 the external heartbeat detected three
5 * production outages perfectly and paged no one. Detection without verified
6 * delivery is theater. This module makes the pager itself a monitored
7 * component: a manual "page me now" test button (/admin/paging) and an
8 * automated weekly drill that sends a clearly-labeled synthetic page
9 * through the real webhook path.
10 *
11 * HONESTY BOUNDARY — what "verified" means here: we can verify that the
12 * webhook ENDPOINT ACCEPTED the page (an HTTP 2xx from the receiver), not
13 * that a human's phone actually buzzed. ntfy/Pushover/Slack all return 200
14 * before the device is reached. Every surface this module feeds says
15 * exactly that — "accepted", never "delivered". End-to-end confirmation is
16 * the human glancing at their device after pressing the test button.
17 *
18 * THE IRONY CHAIN (stated because it is load-bearing): a broken pager
19 * cannot page about itself. If the weekly drill finds the webhook REFUSED
20 * or UNCONFIGURED, sending an alert about that through the same webhook
21 * would vanish into the same hole. So the drill result ALSO lands as a
22 * `doctor:paging` red row through the existing synthetic_checks pipeline —
23 * which renders on /status and /admin/spine, surfaces the next time the
24 * owner looks anywhere — AND the healthchecks dead-man (external, off-box)
25 * covers total silence: if the whole autopilot stops ticking, nothing
26 * sends the dead-man ping and the external service pages independently.
27 * Three legs: webhook (loud), red row (visible), dead-man (external).
28 *
29 * Persistence: NO new tables. Every send is recorded in the existing
30 * audit log under action "paging.drill" (metadata carries kind / severity /
31 * accepted / reason / statusCode), and history is read back the same way
32 * /admin/spine reads its remediation ledger. The weekly guard hydrates
33 * from those audit rows so a deploy (which restarts the process and zeroes
34 * module state) does not re-fire the drill — the exact bug class the spike
35 * cooldown in spine-alert-fanout.ts was patched for on 2026-08-19.
36 */
37
38import { and, desc, eq, gte } from "drizzle-orm";
39import { db } from "../db";
40import { auditLog } from "../db/schema";
41import { audit } from "./notify";
42import {
43 postAlertWebhook,
44 alertSeverity,
45 type AlertSeverity,
46 type PostAlertWebhookDeps,
47 type PostAlertWebhookResult,
48 type SpineAlert,
49} from "./spine-alert-fanout";
50import {
51 persistChecks,
52 type SyntheticCheckResult,
53} from "./synthetic-monitor";
54
55export type TestPageKind = "manual" | "drill";
56
57/** Audit-log action every send is recorded under. */
58export const PAGING_AUDIT_ACTION = "paging.drill";
59
60/** synthetic_checks row name for the drill's red/green verdict. */
61export const PAGING_CHECK_NAME = "doctor:paging";
62
63/** One drill per 7 days. */
64export const PAGING_DRILL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000;
65
66/**
67 * Severity for the weekly drill alternates week to week so BOTH priority
68 * paths a pager distinguishes (ntfy Priority 4 vs 3, and whatever the
69 * receiver maps "warning" vs "info" to) get exercised over time. A drill
70 * that only ever tests one priority proves half the pager. Week parity is
71 * epoch-based so the alternation survives restarts without any state.
72 */
73export function drillSeverityForWeek(nowMs: number): "warning" | "info" {
74 return Math.floor(nowMs / PAGING_DRILL_INTERVAL_MS) % 2 === 0
75 ? "warning"
76 : "info";
77}
78
79/**
80 * postAlertWebhook derives severity from `alert.kind` via alertSeverity()
81 * — there is no direct severity input, and spine-alert-fanout.ts is
82 * read-only for this module. So the test page borrows the existing kind
83 * that maps to the severity we want: "check_recovered" → info,
84 * "error_spike" → warning. The kind leaks into the webhook payload and
85 * the ntfy Tags header, which is acceptable because the title says
86 * PAGING TEST in capitals before anything else.
87 */
88export function kindForSeverity(
89 severity: "warning" | "info"
90): SpineAlert["kind"] {
91 return severity === "warning" ? "error_spike" : "check_recovered";
92}
93
94export interface BuiltTestPage {
95 alert: SpineAlert;
96 severity: AlertSeverity;
97}
98
99/**
100 * Build the clearly-labeled test alert. Exported for tests — the shape is
101 * the contract: a receiver filtering on "PAGING TEST" must always match.
102 */
103export function buildTestPageAlert(
104 kind: TestPageKind,
105 now: Date = new Date()
106): BuiltTestPage {
107 const severity: "warning" | "info" =
108 kind === "manual" ? "info" : drillSeverityForWeek(now.getTime());
109 const alert: SpineAlert = {
110 kind: kindForSeverity(severity),
111 title: "PAGING TEST — reply not required",
112 body:
113 kind === "manual"
114 ? `Manual test page, requested from /admin/paging at ${now.toISOString()}. ` +
115 `If you are reading this on your device, delivery works end to end. No action required.`
116 : `Weekly paging drill (automated) at ${now.toISOString()}, severity ${severity}. ` +
117 `This proves the paging path still accepts pages. No action required.`,
118 url: "/admin/paging",
119 };
120 return { alert, severity: alertSeverity(alert.kind) };
121}
122
123export interface SendTestPageDeps {
124 /** Override the webhook leg (DI for tests). Defaults to postAlertWebhook. */
125 post?: (
126 alert: SpineAlert,
127 deps?: PostAlertWebhookDeps
128 ) => Promise<PostAlertWebhookResult>;
129 /** Forwarded to postAlertWebhook — url/fetchImpl/timeoutMs/origin/now. */
130 webhook?: PostAlertWebhookDeps;
131 /** Override the audit sink (DI for tests). Defaults to audit(). */
132 auditFn?: typeof audit;
133 now?: () => Date;
134}
135
136/**
137 * Send one clearly-labeled test page through the real webhook path and
138 * record the outcome in the audit log. Never throws — the caller reads
139 * `{sent, reason, statusCode}` and renders it honestly.
140 *
141 * The audit metadata deliberately does NOT include the webhook URL: the
142 * audit log renders on admin surfaces and the URL is a capability (anyone
143 * holding an ntfy topic URL can page the owner). Presence is recorded as
144 * a boolean, never the value.
145 */
146export async function sendTestPage(
147 kind: TestPageKind,
148 deps: SendTestPageDeps = {}
149): Promise<PostAlertWebhookResult> {
150 const now = deps.now ?? (() => new Date());
151 const post = deps.post ?? postAlertWebhook;
152 const auditFn = deps.auditFn ?? audit;
153 const at = now();
154 const { alert, severity } = buildTestPageAlert(kind, at);
155
156 let result: PostAlertWebhookResult;
157 try {
158 result = await post(alert, deps.webhook);
159 } catch (err) {
160 // postAlertWebhook never throws, but an injected post has no such
161 // guarantee and a paging TEST must never crash its caller.
162 result = {
163 sent: false,
164 reason: err instanceof Error ? err.message : String(err),
165 };
166 }
167
168 try {
169 await auditFn({
170 action: PAGING_AUDIT_ACTION,
171 targetType: "paging_webhook",
172 metadata: {
173 kind,
174 severity,
175 // "accepted", not "delivered" — see the module header.
176 accepted: result.sent,
177 reason: result.reason ?? null,
178 statusCode: result.statusCode ?? null,
179 at: at.toISOString(),
180 },
181 });
182 } catch (err) {
183 console.error("[paging-drill] audit write failed:", err);
184 }
185
186 return result;
187}
188
189// ---------------------------------------------------------------------------
190// History — read back through the audit log, same query shape as
191// /admin/spine's remediation ledger.
192// ---------------------------------------------------------------------------
193
194export interface PagingDrillHistoryRow {
195 kind: TestPageKind | "unknown";
196 severity: string;
197 accepted: boolean;
198 reason: string | null;
199 statusCode: number | null;
200 createdAt: Date;
201}
202
203/**
204 * How far back history looks. Bounds the query through the created_at
205 * index (audit_log has no action index; an unbounded backward scan pays
206 * in proportion to how quiet the pager has been — the admin-spine
207 * remediation ledger bounds for the same reason). 90 days ≈ 13 drills,
208 * comfortably more than the 10 the page renders.
209 */
210export const PAGING_HISTORY_WINDOW_MS = 90 * 24 * 60 * 60 * 1000;
211
212/** Last N sends, newest first. Empty array on any failure. */
213export async function readPagingDrillHistory(
214 limit = 10
215): Promise<PagingDrillHistoryRow[]> {
216 try {
217 const since = new Date(Date.now() - PAGING_HISTORY_WINDOW_MS);
218 const rows = await db
219 .select({
220 metadata: auditLog.metadata,
221 createdAt: auditLog.createdAt,
222 })
223 .from(auditLog)
224 .where(
225 and(
226 eq(auditLog.action, PAGING_AUDIT_ACTION),
227 gte(auditLog.createdAt, since)
228 )
229 )
230 .orderBy(desc(auditLog.createdAt))
231 .limit(limit);
232 return rows.map((r) => {
233 let meta: Record<string, unknown> = {};
234 try {
235 meta = r.metadata ? JSON.parse(r.metadata) : {};
236 } catch {
237 // A malformed row renders as unknown rather than hiding the send.
238 }
239 return {
240 kind:
241 meta.kind === "manual" || meta.kind === "drill"
242 ? meta.kind
243 : ("unknown" as const),
244 severity: typeof meta.severity === "string" ? meta.severity : "?",
245 accepted: meta.accepted === true,
246 reason: typeof meta.reason === "string" ? meta.reason : null,
247 statusCode:
248 typeof meta.statusCode === "number" ? meta.statusCode : null,
249 createdAt: r.createdAt,
250 };
251 });
252 } catch (err) {
253 console.error("[paging-drill] history query failed:", err);
254 return [];
255 }
256}
257
258// ---------------------------------------------------------------------------
259// Weekly drill — autopilot task body.
260// ---------------------------------------------------------------------------
261
262/**
263 * Once-per-7-days guard, house pattern (module-level lastRanAt like
264 * SMART_DIGEST) — but hydrated from the audit log on the first tick of a
265 * fresh process. Without hydration every deploy would zero the guard and
266 * re-page, and this platform deploys many times a day.
267 */
268let _lastPagingDrillAt = 0;
269let _pagingDrillHydrated = false;
270
271/** Test seam. */
272export function __resetPagingDrillForTests(): void {
273 _lastPagingDrillAt = 0;
274 _pagingDrillHydrated = false;
275}
276
277/** When did the last DRILL send happen, per the audit log? 0 if never. */
278async function loadLastDrillAtFromAudit(): Promise<number> {
279 const history = await readPagingDrillHistory(10);
280 const drill = history.find((h) => h.kind === "drill");
281 return drill ? drill.createdAt.getTime() : 0;
282}
283
284export interface WeeklyPagingDrillDeps {
285 now?: () => number;
286 /** Override the sender (DI for tests). */
287 send?: (kind: TestPageKind) => Promise<PostAlertWebhookResult>;
288 /** Override the synthetic_checks persist (DI for tests). */
289 persist?: (results: SyntheticCheckResult[]) => Promise<void>;
290 /** Override the guard hydration source (DI for tests). */
291 loadLastDrillAt?: () => Promise<number>;
292}
293
294export interface WeeklyPagingDrillSummary {
295 /** false = the 7-day guard held; nothing was sent. */
296 ran: boolean;
297 accepted?: boolean;
298 reason?: string;
299}
300
301/**
302 * The weekly drill: send the synthetic page, then persist the verdict as a
303 * `doctor:paging` row through the synthetic_checks pipeline. A REFUSED or
304 * UNCONFIGURED webhook is a RED spine condition — see the module header
305 * for why the red row (plus the external dead-man) is what carries that
306 * news, not another page through the broken pager. An accepted send
307 * persists green so /status shows the check alive and recoveries render.
308 */
309export async function weeklyPagingDrill(
310 deps: WeeklyPagingDrillDeps = {}
311): Promise<WeeklyPagingDrillSummary> {
312 const now = deps.now ?? Date.now;
313 const nowMs = now();
314
315 if (nowMs - _lastPagingDrillAt < PAGING_DRILL_INTERVAL_MS) {
316 return { ran: false };
317 }
318 if (!_pagingDrillHydrated) {
319 // Once per process: a restart must not forget a drill it already ran.
320 _pagingDrillHydrated = true;
321 try {
322 const persisted = await (deps.loadLastDrillAt ??
323 loadLastDrillAtFromAudit)();
324 if (persisted > _lastPagingDrillAt) _lastPagingDrillAt = persisted;
325 } catch (err) {
326 // Degrades to "no memory" — one extra drill page, not a crash.
327 console.error("[paging-drill] guard hydration failed:", err);
328 }
329 if (nowMs - _lastPagingDrillAt < PAGING_DRILL_INTERVAL_MS) {
330 return { ran: false };
331 }
332 }
333 _lastPagingDrillAt = nowMs;
334
335 const send = deps.send ?? ((k: TestPageKind) => sendTestPage(k));
336 const result = await send("drill");
337
338 const persist = deps.persist ?? persistChecks;
339 const row: SyntheticCheckResult = result.sent
340 ? { name: PAGING_CHECK_NAME, status: "green", durationMs: 0 }
341 : {
342 name: PAGING_CHECK_NAME,
343 status: "red",
344 durationMs: 0,
345 statusCode: result.statusCode,
346 error: `paging drill: webhook ${
347 result.reason ?? "refused"
348 } — pages are NOT reaching the owner`,
349 };
350 try {
351 await persist([row]);
352 } catch (err) {
353 console.error("[paging-drill] check persist failed:", err);
354 }
355
356 return { ran: true, accepted: result.sent, reason: result.reason };
357}
Addedsrc/routes/admin-paging.tsx+449−0View fileUnifiedSplit
@@ -0,0 +1,449 @@
1/**
2 * /admin/paging — the paging test button + drill history (blueprint §7).
3 *
4 * GET /admin/paging — channel state, test button, drill history,
5 * escalation policy (docs/ESCALATION-POLICY.md)
6 * POST /admin/paging/test — send a manual test page NOW, show the result
7 *
8 * Paging you can't test is paging you don't have: on 2026-08-22 the
9 * platform detected three outages perfectly and paged no one. This page
10 * exists so the owner can prove the path with one click instead of
11 * waiting for the next outage to find out.
12 *
13 * Honesty rules, enforced here by construction:
14 * - The webhook URL is a capability (an ntfy topic URL lets anyone page
15 * the owner) and is NEVER printed — presence only. webhookPresence()
16 * returns a boolean-only shape so the value cannot reach JSX; the
17 * paging-drill test suite pins this.
18 * - Results say "accepted", never "delivered": a 2xx proves the webhook
19 * ENDPOINT took the page, not that a phone buzzed. Copy on every
20 * result tells the operator to confirm on their device.
21 *
22 * Gating + idiom copied from admin-ops.tsx: softAuth + isSiteAdmin, flash
23 * messages via redirect query params, scoped `.paging-` CSS so nothing
24 * bleeds into other admin surfaces. Design tokens only — no raw palette.
25 */
26
27import { Hono } from "hono";
28import { Layout } from "../views/layout";
29import { AdminShell } from "../views/admin-shell";
30import { softAuth } from "../middleware/auth";
31import type { AuthEnv } from "../middleware/auth";
32import { isSiteAdmin } from "../lib/admin";
33import {
34 sendTestPage,
35 readPagingDrillHistory,
36 PAGING_DRILL_INTERVAL_MS,
37 type PagingDrillHistoryRow,
38} from "../lib/paging-drill";
39import { relativeTime } from "./admin-deploys-page";
40
41// ---------------------------------------------------------------------------
42// Channel state — presence only, never the value.
43// ---------------------------------------------------------------------------
44
45export interface WebhookPresence {
46 configured: boolean;
47}
48
49/**
50 * Boolean-only by design: this is the ONLY place the route touches
51 * MONITOR_ALERT_WEBHOOK_URL, and it returns no string field at all, so no
52 * render path can leak the URL. Do not add the value to this shape.
53 */
54export function webhookPresence(env: NodeJS.ProcessEnv = process.env): WebhookPresence {
55 return { configured: Boolean((env.MONITOR_ALERT_WEBHOOK_URL ?? "").trim()) };
56}
57
58/** Escalation policy text, or null when the doc is missing. */
59export async function readEscalationPolicy(): Promise<string | null> {
60 try {
61 const f = Bun.file("docs/ESCALATION-POLICY.md");
62 if (!(await f.exists())) return null;
63 const text = await f.text();
64 return text.trim() ? text : null;
65 } catch (err) {
66 console.error("[admin-paging] escalation policy read failed:", err);
67 return null;
68 }
69}
70
71// ---------------------------------------------------------------------------
72// Scoped styles — tokens only.
73// ---------------------------------------------------------------------------
74
75const pagingStyles = `
76 .paging-wrap { max-width: 960px; margin: 0 auto; padding: var(--space-6) var(--space-4); }
77
78 .paging-hero {
79 margin-bottom: var(--space-5);
80 padding: var(--space-5) var(--space-6);
81 background: var(--bg-elevated);
82 border: 1px solid var(--border);
83 border-radius: 16px;
84 }
85 .paging-title {
86 font-size: clamp(26px, 4vw, 36px);
87 font-family: var(--font-display);
88 font-weight: 800;
89 letter-spacing: -0.028em;
90 line-height: 1.05;
91 margin: 0 0 var(--space-2);
92 color: var(--text-strong);
93 }
94 .paging-sub { font-size: 14.5px; color: var(--text-muted); margin: 0; line-height: 1.55; max-width: 640px; }
95
96 .paging-banner {
97 margin-bottom: var(--space-4);
98 padding: 10px 14px;
99 border-radius: 10px;
100 font-size: 13.5px;
101 border: 1px solid var(--border);
102 background: var(--bg-elevated);
103 color: var(--text);
104 display: flex;
105 align-items: center;
106 gap: 10px;
107 line-height: 1.5;
108 }
109 .paging-banner.is-ok {
110 border-color: color-mix(in srgb, var(--green) 45%, transparent);
111 background: color-mix(in srgb, var(--green) 9%, transparent);
112 }
113 .paging-banner.is-error {
114 border-color: color-mix(in srgb, var(--red) 45%, transparent);
115 background: color-mix(in srgb, var(--red) 8%, transparent);
116 }
117 .paging-banner-dot { width: 8px; height: 8px; border-radius: 9999px; flex-shrink: 0; background: var(--text-muted); }
118 .paging-banner.is-ok .paging-banner-dot { background: var(--green); }
119 .paging-banner.is-error .paging-banner-dot { background: var(--red); }
120
121 .paging-section {
122 margin-bottom: var(--space-5);
123 background: var(--bg-elevated);
124 border: 1px solid var(--border);
125 border-radius: 14px;
126 overflow: hidden;
127 }
128 .paging-section-head { padding: var(--space-4) var(--space-5); border-bottom: 1px solid var(--border); }
129 .paging-section-title {
130 margin: 0;
131 font-family: var(--font-display);
132 font-size: 17px;
133 font-weight: 700;
134 letter-spacing: -0.018em;
135 color: var(--text-strong);
136 }
137 .paging-section-sub { margin: 6px 0 0; font-size: 12.5px; color: var(--text-muted); line-height: 1.5; }
138 .paging-section-body { padding: var(--space-4) var(--space-5); }
139
140 .paging-pill {
141 display: inline-flex;
142 align-items: center;
143 gap: 6px;
144 padding: 3px 10px;
145 border-radius: 9999px;
146 font-size: 11.5px;
147 font-weight: 600;
148 }
149 .paging-pill .dot { width: 6px; height: 6px; border-radius: 9999px; background: currentColor; }
150 .paging-pill.is-ok { background: color-mix(in srgb, var(--green) 15%, transparent); color: var(--green); }
151 .paging-pill.is-bad { background: color-mix(in srgb, var(--red) 14%, transparent); color: var(--red); }
152 .paging-pill.is-muted { background: var(--bg-tertiary); color: var(--text-muted); }
153
154 .paging-btn {
155 display: inline-flex;
156 align-items: center;
157 gap: 6px;
158 padding: 9px 16px;
159 border-radius: 10px;
160 font-size: 13px;
161 font-weight: 600;
162 border: 1px solid transparent;
163 cursor: pointer;
164 font: inherit;
165 line-height: 1;
166 background: var(--accent);
167 color: var(--bg);
168 transition: transform 120ms ease, opacity 120ms ease;
169 }
170 .paging-btn:hover:not(:disabled) { transform: translateY(-1px); }
171 .paging-btn:disabled { cursor: not-allowed; opacity: 0.5; transform: none; }
172 .paging-hint { font-size: 12px; color: var(--text-muted); line-height: 1.5; max-width: 480px; }
173 .paging-actions { display: flex; align-items: center; gap: var(--space-3); flex-wrap: wrap; }
174 .paging-actions form { margin: 0; }
175
176 .paging-table { width: 100%; border-collapse: collapse; font-size: 13px; }
177 .paging-table th {
178 text-align: left;
179 padding: 8px 10px;
180 font-size: 11px;
181 text-transform: uppercase;
182 letter-spacing: 0.06em;
183 color: var(--text-muted);
184 border-bottom: 1px solid var(--border);
185 font-weight: 600;
186 }
187 .paging-table td { padding: 9px 10px; border-bottom: 1px solid var(--border); color: var(--text); vertical-align: top; }
188 .paging-table tr:last-child td { border-bottom: none; }
189 .paging-table .reason { font-family: var(--font-mono); font-size: 12px; color: var(--text-muted); word-break: break-word; }
190 .paging-empty { font-size: 13px; color: var(--text-muted); padding: var(--space-3) 0; }
191
192 .paging-policy {
193 margin: 0;
194 padding: var(--space-3) 0 0;
195 font-size: 13px;
196 line-height: 1.65;
197 color: var(--text);
198 white-space: pre-wrap;
199 word-break: break-word;
200 font-family: var(--font-mono);
201 overflow-x: auto;
202 }
203
204 .paging-403 {
205 max-width: 540px;
206 margin: var(--space-12) auto;
207 padding: var(--space-6);
208 text-align: center;
209 background: var(--bg-elevated);
210 border: 1px solid var(--border);
211 border-radius: 16px;
212 }
213 .paging-403 h2 { font-family: var(--font-display); font-size: 22px; margin: 0 0 8px; color: var(--text-strong); }
214 .paging-403 p { color: var(--text-muted); margin: 0; font-size: 14px; }
215`;
216
217// ---------------------------------------------------------------------------
218// Gating
219// ---------------------------------------------------------------------------
220
221const paging = new Hono<AuthEnv>();
222paging.use("*", softAuth);
223
224async function gate(c: any): Promise<{ user: any } | Response> {
225 const user = c.get("user");
226 if (!user) return c.redirect("/login?next=/admin/paging");
227 if (!(await isSiteAdmin(user.id))) {
228 return c.html(
229 <Layout title="Forbidden" user={user}>
230 <div class="paging-403">
231 <h2>403 — Not a site admin</h2>
232 <p>You don't have permission to view this page.</p>
233 </div>
234 <style dangerouslySetInnerHTML={{ __html: pagingStyles }} />
235 </Layout>,
236 403
237 );
238 }
239 return { user };
240}
241
242// ---------------------------------------------------------------------------
243// GET /admin/paging
244// ---------------------------------------------------------------------------
245
246function historyRow(h: PagingDrillHistoryRow) {
247 return (
248 <tr>
249 <td title={h.createdAt.toISOString()}>{relativeTime(h.createdAt)}</td>
250 <td>{h.kind}</td>
251 <td>{h.severity}</td>
252 <td>
253 {h.accepted ? (
254 <span class="paging-pill is-ok">
255 <span class="dot" aria-hidden="true" />
256 accepted{h.statusCode ? ` (HTTP ${h.statusCode})` : ""}
257 </span>
258 ) : (
259 <span class="paging-pill is-bad">
260 <span class="dot" aria-hidden="true" />
261 refused
262 </span>
263 )}
264 {!h.accepted && h.reason && <div class="reason">{h.reason}</div>}
265 </td>
266 </tr>
267 );
268}
269
270paging.get("/admin/paging", async (c) => {
271 const g = await gate(c);
272 if (g instanceof Response) return g;
273 const { user } = g;
274
275 const success = c.req.query("success");
276 const error = c.req.query("error");
277
278 const [presence, history, policy] = await Promise.all([
279 Promise.resolve(webhookPresence()),
280 readPagingDrillHistory(10),
281 readEscalationPolicy(),
282 ]);
283 const drillDays = Math.round(PAGING_DRILL_INTERVAL_MS / 86_400_000);
284
285 return c.html(
286 <AdminShell active="paging" title="Paging" user={user}>
287 <div class="paging-wrap">
288 <section class="paging-hero">
289 <h1 class="paging-title">Paging</h1>
290 <p class="paging-sub">
291 Paging you can't test is paging you don't have. This page sends a
292 clearly-labeled test through the real webhook path and keeps the
293 drill ledger. A green result means the endpoint <em>accepted</em>{" "}
294 the page — only your device proves delivery.
295 </p>
296 </section>
297
298 {success && (
299 <div class="paging-banner is-ok" role="status">
300 <span class="paging-banner-dot" aria-hidden="true" />
301 {decodeURIComponent(success)}
302 </div>
303 )}
304 {error && (
305 <div class="paging-banner is-error" role="alert">
306 <span class="paging-banner-dot" aria-hidden="true" />
307 {decodeURIComponent(error)}
308 </div>
309 )}
310
311 {/* ─── Channel + test button ─── */}
312 <section class="paging-section">
313 <header class="paging-section-head">
314 <h3 class="paging-section-title">Paging channel</h3>
315 <p class="paging-section-sub">
316 MONITOR_ALERT_WEBHOOK_URL — the phone-paging webhook alerts go
317 through first. The URL itself is never shown here: it is a
318 capability, and printing it would let anyone who sees this page
319 page you.
320 </p>
321 </header>
322 <div class="paging-section-body">
323 <p style="margin:0 0 var(--space-3);font-size:13.5px">
324 Webhook:{" "}
325 {presence.configured ? (
326 <span class="paging-pill is-ok">
327 <span class="dot" aria-hidden="true" />
328 configured
329 </span>
330 ) : (
331 <span class="paging-pill is-bad">
332 <span class="dot" aria-hidden="true" />
333 not configured — nothing can page your phone
334 </span>
335 )}
336 </p>
337 <div class="paging-actions">
338 <form method="post" action="/admin/paging/test">
339 <button
340 type="submit"
341 class="paging-btn"
342 disabled={!presence.configured}
343 title={
344 presence.configured
345 ? "Send a test page through the real webhook path"
346 : "Set MONITOR_ALERT_WEBHOOK_URL first (see .env.example)"
347 }
348 >
349 Send test page now
350 </button>
351 </form>
352 <span class="paging-hint">
353 Sends "PAGING TEST — reply not required" at info severity.
354 {presence.configured
355 ? " After it reports accepted, check your device — that last hop is the part no dashboard can verify."
356 : " Configure MONITOR_ALERT_WEBHOOK_URL in /opt/gluecron/.env (an ntfy topic URL works) to enable this."}
357 </span>
358 </div>
359 </div>
360 </section>
361
362 {/* ─── Drill history ─── */}
363 <section class="paging-section">
364 <header class="paging-section-head">
365 <h3 class="paging-section-title">Send history</h3>
366 <p class="paging-section-sub">
367 Every test and drill, from the audit log. The automated drill
368 runs every {drillDays} days and alternates severity so both
369 priority paths get exercised; a refused drill goes red on{" "}
370 <a href="/admin/spine" style="color:var(--accent)">/admin/spine</a>{" "}
371 and <a href="/status" style="color:var(--accent)">/status</a> as{" "}
372 <code>doctor:paging</code>.
373 </p>
374 </header>
375 <div class="paging-section-body">
376 {history.length === 0 ? (
377 <p class="paging-empty">
378 No pages sent yet. Press the test button above — the first row
379 lands here, and the weekly drill will follow on its own.
380 </p>
381 ) : (
382 <table class="paging-table">
383 <thead>
384 <tr>
385 <th>When</th>
386 <th>Kind</th>
387 <th>Severity</th>
388 <th>Endpoint result</th>
389 </tr>
390 </thead>
391 <tbody>{history.map(historyRow)}</tbody>
392 </table>
393 )}
394 </div>
395 </section>
396
397 {/* ─── Escalation policy ─── */}
398 <section class="paging-section">
399 <header class="paging-section-head">
400 <h3 class="paging-section-title">Escalation policy</h3>
401 <p class="paging-section-sub">
402 Rendered from <code>docs/ESCALATION-POLICY.md</code> at page
403 load, so this page can never show a stale copy of the policy.
404 </p>
405 </header>
406 <div class="paging-section-body">
407 {policy ? (
408 <pre class="paging-policy">{policy}</pre>
409 ) : (
410 <p class="paging-empty">
411 docs/ESCALATION-POLICY.md is missing from this deployment —
412 there is no written escalation policy on file. That is a gap,
413 not a formatting problem: write one and commit it.
414 </p>
415 )}
416 </div>
417 </section>
418 </div>
419 <style dangerouslySetInnerHTML={{ __html: pagingStyles }} />
420 </AdminShell>
421 );
422});
423
424// ---------------------------------------------------------------------------
425// POST /admin/paging/test
426// ---------------------------------------------------------------------------
427
428paging.post("/admin/paging/test", async (c) => {
429 const g = await gate(c);
430 if (g instanceof Response) return g;
431
432 const result = await sendTestPage("manual");
433 if (result.sent) {
434 // "accepted", not "delivered" — the 2xx proves the endpoint took the
435 // page; only the operator's device proves the rest of the chain.
436 return c.redirect(
437 `/admin/paging?success=${encodeURIComponent(
438 `Webhook accepted the test page (HTTP ${result.statusCode ?? "2xx"}) — now confirm it reached your device.`
439 )}`
440 );
441 }
442 return c.redirect(
443 `/admin/paging?error=${encodeURIComponent(
444 `Test page was NOT accepted: ${result.reason ?? "unknown"}. Nothing reached your device.`
445 )}`
446 );
447});
448
449export default paging;
Modifiedsrc/views/admin-shell.tsx+2−0View fileUnifiedSplit
@@ -27,6 +27,7 @@ export type AdminNavKey =
2727 | "truth"
2828 | "errors"
2929 | "ops"
30 | "paging"
3031 | "health"
3132 | "env-health"
3233 | "status"
@@ -96,6 +97,7 @@ export const ADMIN_NAV: AdminNavGroup[] = [
9697 // you what actually threw.
9798 { key: "errors", href: "/admin/errors", label: "Errors" },
9899 { key: "ops", href: "/admin/ops", label: "Operations" },
100 { key: "paging", href: "/admin/paging", label: "Paging" },
99101 { key: "health", href: "/admin/health", label: "Health (traffic lights)" },
100102 { key: "env-health", href: "/admin/env-health", label: "Env / feature health" },
101103 { key: "status", href: "/admin/status", label: "Platform monitor" },
102104
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts