fix(status): a sibling product's outage is not our outage — estate checks leave the uptime math #5543
3 changed files+145−4
Modifiedsrc/__tests__/incident-recorder.test.ts+63−0View fileUnifiedSplit
@@ -328,4 +328,67 @@ describe("syncIncidentsFromChecks", () => {
328328 expect(out.action).toBe("none");
329329 expect(out.error).toBe("db is down");
330330 });
331
332 // ── estate scoping (2026-08-27) ──────────────────────────────────────
333 //
334 // `estate:*` checks probe sibling products' domains. Before the filter,
335 // gatetest.ai's expired domain filed a platform incident that dragged
336 // the public 24h uptime to 56.6% and painted every service "Degraded"
337 // while the platform served every request.
338
339 it("an estate check red for two ticks files NO platform incident", async () => {
340 const { deps, calls } = harness(null);
341 const results = [
342 red("estate:gatetest.ai", "Could not resolve host"),
343 green("healthz"),
344 green("login"),
345 ];
346
347 const out = await syncIncidentsFromChecks(results, prevOf(results), deps);
348
349 expect(out.action).toBe("none");
350 expect(calls.opened).toHaveLength(0);
351 });
352
353 it("all-platform-green resolves an open incident even while estate is red", async () => {
354 const openRow: OpenIncidentRow = {
355 id: "inc-estate-legacy",
356 severity: "minor",
357 status: "investigating",
358 title: "estate:gatetest.ai is unavailable",
359 body: "b",
360 };
361 const { deps, calls } = harness(openRow);
362 const results = [
363 red("estate:gatetest.ai", "Could not resolve host"),
364 green("healthz"),
365 green("login"),
366 ];
367
368 const out = await syncIncidentsFromChecks(results, prevOf(results), deps);
369
370 expect(out.action).toBe("resolved");
371 expect(calls.resolved).toHaveLength(1);
372 });
373
374 it("estate reds never inflate a real platform incident's severity", async () => {
375 const { deps, calls } = harness(null);
376 // 1 platform check red out of 2 platform checks + 3 estate reds.
377 // Counted together the proportional rule would scream critical;
378 // scoped correctly this is one major.
379 const results = [
380 red("healthz"),
381 green("login"),
382 red("estate:a.com"),
383 red("estate:b.com"),
384 red("estate:c.com"),
385 ];
386
387 const out = await syncIncidentsFromChecks(results, prevOf(results), deps);
388
389 expect(out.action).toBe("opened");
390 expect(out.severity).toBe("major");
391 expect(calls.opened[0].title).toBe("Health endpoint is unavailable");
392 expect(calls.opened[0].body).not.toContain("estate:");
393 });
331394});
Modifiedsrc/lib/incident-recorder.ts+17−4View fileUnifiedSplit
@@ -353,8 +353,21 @@ export async function syncIncidentsFromChecks(
353353 if (results.length === 0) return empty;
354354
355355 try {
356 const failing = confirmedFailures(results, previous);
357 const anyRedNow = results.some((r) => r.status === "red");
356 // `estate:*` checks probe OTHER Gluecron-operated products' domains
357 // (alecrae.com, gatetest.ai, …). They ride the same monitor for its
358 // transition alerting, but a sibling product's outage is not
359 // gluecron.com's outage — the same doctrine that already keeps
360 // `surface:`/`dep:`/`doctor:` rows out of public incidents. Before this
361 // filter, gatetest.ai's expired domain filed a platform incident that
362 // dragged the public 24h uptime to 56.6% and painted Database/API/Git
363 // "Degraded" while all of them served every request (2026-08-27).
364 // Estate reds still alert through the spine and still render on
365 // /status — in their own section, outside the uptime math.
366 const platformResults = results.filter(
367 (r) => !r.name.startsWith("estate:")
368 );
369 const failing = confirmedFailures(platformResults, previous);
370 const anyRedNow = platformResults.some((r) => r.status === "red");
358371 const open = await findOpen();
359372
360373 // ── Recovery: everything green, so close anything still open ──
@@ -382,9 +395,9 @@ export async function syncIncidentsFromChecks(
382395 return { ...empty, action: "none" };
383396 }
384397
385 const severity = classifySeverity(failing, results.length);
398 const severity = classifySeverity(failing, platformResults.length);
386399 const title = incidentTitle(failing);
387 const body = incidentBody(failing, results);
400 const body = incidentBody(failing, platformResults);
388401
389402 // ── No open incident: file one ──
390403 if (!open) {
Modifiedsrc/routes/status.tsx+65−0View fileUnifiedSplit
@@ -374,6 +374,21 @@ status.get("/status", async (c) => {
374374 recentIncidentRows = [];
375375 }
376376
377 // `estate:*` rows are probes of OTHER Gluecron-operated products
378 // (alecrae.com, gatetest.ai, …) run from this box. Same doctrine as the
379 // doctor: filter above: a sibling product's outage is real information,
380 // but it is not gluecron.com's outage — before this split, gatetest.ai's
381 // expired domain flipped this page to "Degraded performance" and dragged
382 // every service row to 56.6% while the platform served every request.
383 // Estate reds get their own section below, outside the uptime math and
384 // the headline.
385 const estateRedRows = recentIncidentRows.filter((r) =>
386 r.name.startsWith("estate:")
387 );
388 recentIncidentRows = recentIncidentRows.filter(
389 (r) => !r.name.startsWith("estate:")
390 );
391
377392 // ONE query feeds every incident-derived figure on the page: the 24h /
378393 // 30d / 90d uptime windows, "days since last incident", the headline
379394 // (any unresolved row) and the history table (first 10). Bounded to the
@@ -418,6 +433,15 @@ status.get("/status", async (c) => {
418433 } catch {
419434 incidentWindow = [];
420435 }
436 // Estate-scoped incidents filed before the recorder learned to keep
437 // `estate:*` checks out of platform incidents (2026-08-27) would keep
438 // wounding the 24h/30d uptime figures until they aged out of the 90-day
439 // window. Single-check estate incidents are identifiable by title; they
440 // are sibling-product outages, not ours, so they leave the uptime math
441 // and the platform history entirely.
442 incidentWindow = incidentWindow.filter(
443 (i) => !i.title.startsWith("estate:")
444 );
421445 const incidentHistory: IncidentRow[] = incidentWindow.slice(0, 10);
422446
423447 const overallOk =
@@ -983,6 +1007,47 @@ status.get("/status", async (c) => {
9831007 </div>
9841008 </section>
9851009
1010 {/* ─── Estate watch — sibling products, never platform uptime ─── */}
1011 {estateRedRows.length > 0 && (
1012 <section class="status-section" aria-labelledby="status-estate-h">
1013 <header class="status-section-head">
1014 <div>
1015 <p class="status-section-eyebrow">Estate</p>
1016 <h2 class="status-section-title" id="status-estate-h">
1017 Estate watch
1018 </h2>
1019 <p class="status-section-sub">
1020 Other Gluecron-operated products, probed from this box.
1021 Their outages are shown for honesty but are not
1022 gluecron.com incidents and never count toward the uptime
1023 figures above.
1024 </p>
1025 </div>
1026 <span class="status-count-pill is-warn">
1027 {estateRedRows.length} red
1028 </span>
1029 </header>
1030 <div class="status-section-body">
1031 <ul class="status-alert-list">
1032 {estateRedRows.map((r) => (
1033 <li class="status-alert-row">
1034 <span class="status-alert-dot" aria-hidden="true" />
1035 <div class="status-alert-main">
1036 <p class="status-alert-name">
1037 <code>{r.name}</code>
1038 </p>
1039 <p class="status-alert-err">
1040 {r.error || "(no error message)"}
1041 </p>
1042 </div>
1043 <time class="status-inc-time">{fmtDate(r.checkedAt)}</time>
1044 </li>
1045 ))}
1046 </ul>
1047 </div>
1048 </section>
1049 )}
1050
9861051 {/* ─── Platform stats ─── */}
9871052 <section class="status-section" aria-labelledby="status-stats-h">
9881053 <header class="status-section-head">
9891054
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts