feat(spine): red probes and error spikes now REACH the owner #5471
3 changed files+316−0
Addedsrc/__tests__/spine-alert-fanout.test.ts+119−0View fileUnifiedSplit
@@ -0,0 +1,119 @@
1/**
2 * Red probes must REACH the owner, not just render.
3 *
4 * The transition detector in runSyntheticMonitorTaskOnce existed and
5 * worked — and delivered every alert to MONITOR_ALERT_WEBHOOK_URL, an env
6 * var nothing sets in production. Detection without delivery: the exact
7 * "partial coverage that renders as complete" trap, in the monitoring
8 * layer itself. These tests pin the owner fan-out path: batched, edge-
9 * triggered, recovery-notifying.
10 */
11
12import { describe, expect, it } from "bun:test";
13import { runSyntheticMonitorTaskOnce } from "../lib/autopilot";
14import type { SyntheticCheckResult } from "../lib/synthetic-monitor";
15import type { SpineAlert } from "../lib/spine-alert-fanout";
16
17function result(
18 name: string,
19 status: SyntheticCheckResult["status"],
20 error?: string
21): SyntheticCheckResult {
22 return { name, status, durationMs: 5, error };
23}
24
25async function runTick(opts: {
26 previous: Record<string, SyntheticCheckResult>;
27 current: SyntheticCheckResult[];
28}): Promise<SpineAlert[]> {
29 const alerts: SpineAlert[] = [];
30 await runSyntheticMonitorTaskOnce({
31 runChecks: async () => opts.current,
32 persist: async () => {},
33 loadPrevious: async () => opts.previous,
34 postAlert: async () => {},
35 alertUrl: () => "",
36 syncIncidents: async () => ({ action: "none" }) as never,
37 fanOut: async (a) => {
38 alerts.push(a);
39 },
40 });
41 // Spike detection also runs and may push an error_spike (or, in tests
42 // with no DB, swallow its failure and push nothing) — callers filter.
43 return alerts;
44}
45
46describe("spine owner fan-out", () => {
47 it("batches every green→red transition into ONE alert", async () => {
48 const alerts = await runTick({
49 previous: {
50 a: result("a", "green"),
51 b: result("b", "green"),
52 c: result("c", "green"),
53 },
54 current: [
55 result("a", "red", "timeout after 5000ms"),
56 result("b", "red", "expected status 200, got 502"),
57 result("c", "green"),
58 ],
59 });
60 const red = alerts.filter((a) => a.kind === "check_red");
61 expect(red).toHaveLength(1);
62 expect(red[0]!.title).toContain("2 checks went red");
63 expect(red[0]!.body).toContain("a: timeout");
64 expect(red[0]!.body).toContain("b: expected status");
65 expect(red[0]!.body).not.toContain("- c:");
66 });
67
68 it("red→red repeats stay quiet; red→green sends a recovery", async () => {
69 const alerts = await runTick({
70 previous: {
71 alpha: result("alpha", "red", "still down"),
72 bravo: result("bravo", "red", "was down"),
73 },
74 current: [result("alpha", "red", "still down"), result("bravo", "green")],
75 });
76 expect(alerts.filter((a) => a.kind === "check_red")).toHaveLength(0);
77 const rec = alerts.filter((a) => a.kind === "check_recovered");
78 expect(rec).toHaveLength(1);
79 expect(rec[0]!.body).toContain("bravo");
80 expect(rec[0]!.body).not.toContain("alpha");
81 });
82
83 it("an all-green tick sends nothing", async () => {
84 const alerts = await runTick({
85 previous: { a: result("a", "green") },
86 current: [result("a", "green")],
87 });
88 expect(
89 alerts.filter((a) => a.kind === "check_red" || a.kind === "check_recovered")
90 ).toHaveLength(0);
91 });
92
93 it("a fresh DB (no prior state) going red still alerts", async () => {
94 const alerts = await runTick({
95 previous: {},
96 current: [result("a", "red", "boom")],
97 });
98 expect(alerts.filter((a) => a.kind === "check_red")).toHaveLength(1);
99 });
100});
101
102describe("fan-out delivery is wired to real channels", () => {
103 it("fanOutSpineAlert reaches createNotification AND sendEmail", async () => {
104 const src = await Bun.file("src/lib/spine-alert-fanout.ts").text();
105 expect(src).toContain("createNotification");
106 expect(src).toContain("sendEmail");
107 // Undeliverable alerts must at least hit the server log.
108 expect(src).toContain("UNDELIVERABLE");
109 });
110
111 it("spike detection is bounded, cooled down, and never throws", async () => {
112 const src = await Bun.file("src/lib/spine-alert-fanout.ts").text();
113 const fn = src.slice(src.indexOf("export async function detectAndAlertNewErrorSpike"));
114 expect(fn).toContain("SPIKE_COOLDOWN_MS");
115 expect(fn).toMatch(/\.limit\(\d+\)/);
116 expect(fn).toContain("catch");
117 expect(fn).toContain("return null;");
118 });
119});
Modifiedsrc/lib/autopilot.ts+48−0View fileUnifiedSplit
@@ -1098,6 +1098,8 @@ export interface SyntheticMonitorTaskDeps {
10981098 results: SyntheticCheckResult[],
10991099 previous: Record<string, SyntheticCheckResult>
11001100 ) => Promise<IncidentSyncSummary>;
1101 /** Override the owner fan-out (DI for tests). */
1102 fanOut?: (alert: import("./spine-alert-fanout").SpineAlert) => Promise<void>;
11011103}
11021104
11031105export interface SyntheticMonitorTaskSummary {
@@ -1185,6 +1187,8 @@ export async function runSyntheticMonitorTaskOnce(
11851187 let yellow = 0;
11861188 let transitions = 0;
11871189 const url = alertUrl();
1190 const wentRed: SyntheticCheckResult[] = [];
1191 const recovered: SyntheticCheckResult[] = [];
11881192
11891193 for (const r of results) {
11901194 if (r.status === "green") green += 1;
@@ -1199,6 +1203,7 @@ export async function runSyntheticMonitorTaskOnce(
11991203 const priorWasGreen = !prior || prior.status === "green";
12001204 if (priorWasGreen && r.status === "red") {
12011205 transitions += 1;
1206 wentRed.push(r);
12021207 if (url) {
12031208 await postAlert(url, {
12041209 check: r.name,
@@ -1210,6 +1215,49 @@ export async function runSyntheticMonitorTaskOnce(
12101215 });
12111216 }
12121217 }
1218 if (prior?.status === "red" && r.status === "green") {
1219 recovered.push(r);
1220 }
1221 }
1222
1223 // Owner fan-out (in-app + email) — the webhook above only ever reached
1224 // MONITOR_ALERT_WEBHOOK_URL, which nothing sets in production, so until
1225 // 2026-08-09 every detected transition rendered on /admin/spine and
1226 // reached no human. Batched: a full outage flips ~14 checks in one tick
1227 // and must produce ONE message, not fourteen. Edge-triggered like the
1228 // webhook, so red→red repeats stay quiet.
1229 const fanOut =
1230 deps.fanOut ??
1231 (async (alert: import("./spine-alert-fanout").SpineAlert) => {
1232 const m = await import("./spine-alert-fanout");
1233 await m.fanOutSpineAlert(alert);
1234 });
1235 try {
1236 if (wentRed.length > 0) {
1237 const lines = wentRed
1238 .map((r) => `- ${r.name}: ${r.error ?? `status ${r.statusCode ?? "?"}`}`)
1239 .join("\n");
1240 await fanOut({
1241 kind: "check_red",
1242 title: `🔴 Spine: ${wentRed.length} check${wentRed.length === 1 ? "" : "s"} went red`,
1243 body: `Green→red this tick:\n${lines}`,
1244 });
1245 }
1246 if (recovered.length > 0) {
1247 await fanOut({
1248 kind: "check_recovered",
1249 title: `✅ Spine: ${recovered.length} check${recovered.length === 1 ? "" : "s"} recovered`,
1250 body: `Back to green: ${recovered.map((r) => r.name).join(", ")}`,
1251 });
1252 }
1253 // New-error spike — novel fingerprints clustering is the signature of
1254 // a broken deploy even while every endpoint still answers 200.
1255 const { detectAndAlertNewErrorSpike } = await import(
1256 "./spine-alert-fanout"
1257 );
1258 await detectAndAlertNewErrorSpike(fanOut);
1259 } catch (err) {
1260 console.error("[autopilot] synthetic-monitor: owner fan-out threw:", err);
12131261 }
12141262
12151263 // Reconcile the public incident record. Wrapped defensively even though
Addedsrc/lib/spine-alert-fanout.ts+149−0View fileUnifiedSplit
@@ -0,0 +1,149 @@
1/**
2 * Spine alert fan-out — red probes and error spikes REACH the owner.
3 *
4 * The spine already detected green→red transitions (autopilot's
5 * runSyntheticMonitorTaskOnce) but only posted them to
6 * MONITOR_ALERT_WEBHOOK_URL — an env var nothing sets in production. Every
7 * detection rendered on /admin/spine and reached no human. "Know instantly"
8 * is an alerting property, not a dashboard property; this module is the
9 * mouth the spine didn't have.
10 *
11 * Channels, per site admin (users.is_admin):
12 * - in-app notification (the bell — createNotification)
13 * - email via sendEmail (Resend when configured; degrades to log)
14 *
15 * Callers batch: one alert per tick listing every transition, not one
16 * email per red check — a total outage flips ~14 checks at once and must
17 * produce ONE message, not fourteen.
18 *
19 * Never throws. An alerting failure must not damage the tick that is
20 * already dealing with a broken site.
21 */
22
23import { eq, sql } from "drizzle-orm";
24import { db } from "../db";
25import { users, platformErrors } from "../db/schema";
26import { createNotification } from "./notify";
27import { sendEmail } from "./email";
28
29export interface SpineAlert {
30 kind: "check_red" | "check_recovered" | "error_spike";
31 title: string;
32 body: string;
33 /** Where the notification links. Defaults to /admin/spine. */
34 url?: string;
35}
36
37/** Resolve every site admin's id + email. Empty array on any failure. */
38async function siteAdmins(): Promise<Array<{ id: string; email: string }>> {
39 try {
40 return await db
41 .select({ id: users.id, email: users.email })
42 .from(users)
43 .where(eq(users.isAdmin, true));
44 } catch (err) {
45 console.error("[spine-alert] admin lookup failed:", err);
46 return [];
47 }
48}
49
50export async function fanOutSpineAlert(alert: SpineAlert): Promise<void> {
51 try {
52 const admins = await siteAdmins();
53 if (admins.length === 0) {
54 console.error(
55 `[spine-alert] UNDELIVERABLE (no site admins): ${alert.title}`
56 );
57 return;
58 }
59 const url = alert.url ?? "/admin/spine";
60 for (const admin of admins) {
61 await createNotification({
62 userId: admin.id,
63 type: `spine_${alert.kind}`,
64 title: alert.title,
65 body: alert.body,
66 url,
67 });
68 // Email every kind, including recovery — an outage email with no
69 // recovery email leaves the reader assuming the worst.
70 const res = await sendEmail({
71 to: admin.email,
72 subject: alert.title,
73 text: `${alert.body}\n\nhttps://gluecron.com${url}`,
74 });
75 if (!res.ok && !res.skipped) {
76 console.error(
77 `[spine-alert] email to ${admin.email} failed:`,
78 (res as { error?: string }).error ?? "unknown"
79 );
80 }
81 }
82 } catch (err) {
83 console.error("[spine-alert] fan-out failed:", err);
84 }
85}
86
87/**
88 * NEW-error spike detection: distinct fingerprints first seen inside the
89 * window. Novel errors clustering in time is the signature of a broken
90 * deploy — repeat occurrences of known errors (occurrences bumps on an
91 * existing fingerprint) deliberately do NOT trip this.
92 */
93export const SPIKE_WINDOW_MINUTES = 10;
94export const SPIKE_THRESHOLD = 3;
95/** At most one spike alert per hour — the errors page has the detail. */
96export const SPIKE_COOLDOWN_MS = 60 * 60_000;
97
98let lastSpikeAlertAt = 0;
99
100/** Test seam. */
101export function __resetSpikeCooldownForTests(): void {
102 lastSpikeAlertAt = 0;
103}
104
105/**
106 * Returns the spike alert it sent, or null when quiet / cooling down /
107 * on any DB failure. Called once per synthetic-monitor tick.
108 */
109export async function detectAndAlertNewErrorSpike(
110 fanOut: (a: SpineAlert) => Promise<void> = fanOutSpineAlert,
111 now: number = Date.now()
112): Promise<SpineAlert | null> {
113 if (now - lastSpikeAlertAt < SPIKE_COOLDOWN_MS) return null;
114 try {
115 const rows = await db
116 .select({
117 kind: platformErrors.kind,
118 message: platformErrors.message,
119 path: platformErrors.path,
120 })
121 .from(platformErrors)
122 .where(
123 sql`${platformErrors.firstSeenAt} > now() - interval '${sql.raw(
124 String(SPIKE_WINDOW_MINUTES)
125 )} minutes' AND ${platformErrors.resolvedAt} IS NULL`
126 )
127 .limit(10);
128 if (rows.length < SPIKE_THRESHOLD) return null;
129
130 lastSpikeAlertAt = now;
131 const sample = rows
132 .slice(0, 5)
133 .map((r) => `- ${r.kind}: ${r.message.slice(0, 140)}${r.path ? ` (${r.path})` : ""}`)
134 .join("\n");
135 const alert: SpineAlert = {
136 kind: "error_spike",
137 title: `🔴 Error spike: ${rows.length}+ new error types in ${SPIKE_WINDOW_MINUTES} min`,
138 body:
139 `Distinct NEW error fingerprints in the last ${SPIKE_WINDOW_MINUTES} minutes — ` +
140 `this is the signature of a broken deploy or a failing dependency.\n\n${sample}`,
141 url: "/admin/errors",
142 };
143 await fanOut(alert);
144 return alert;
145 } catch (err) {
146 console.error("[spine-alert] spike detection failed:", err);
147 return null;
148 }
149}
0150
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts