feat(reliability): load-aware CI claim guard + niced steps + dead-man ping hook #5519
4 changed files+104−2
Modified.env.example+5−0View fileUnifiedSplit
@@ -337,3 +337,8 @@ TRUSTED_PROXY_HOPS=1
337337# Largest single LFS object accepted (bytes). Default 512 MiB. Objects are
338338# stored content-addressed under ${GIT_REPOS_PATH}/.lfs/ on the repos volume.
339339LFS_MAX_OBJECT_BYTES=
340
341# CI claim guard: defer claiming runs while 1-min load exceeds this
342# (default: 2x core count). The INCIDENT 2026-08-22 fix — CI can no longer
343# strangle the box it runs on.
344WORKFLOW_LOAD_DEFER=
Modified.github/workflows/heartbeat.yml+29−0View fileUnifiedSplit
@@ -118,6 +118,35 @@ jobs:
118118 echo "down report → HTTP ${code}: $(cat /tmp/hb.out 2>/dev/null | head -c 300)"
119119 true
120120
121 # The channel that works when NOTHING on the box does: ping an
122 # external dead-man service. Detection without paging is what the
123 # 2026-08-22 triple outage proved insufficient — four red runs here,
124 # zero pages. Set the HEALTHCHECK_PING_URL repo secret (healthchecks.io
125 # check URL); /fail pings on a red probe, the bare URL on green.
126 # Skipped with a note when unset; never turns a probe result red.
127 - name: Ping external dead-man (fail)
128 if: failure()
129 env:
130 HEALTHCHECK_PING_URL: ${{ secrets.HEALTHCHECK_PING_URL }}
131 run: |
132 set -u
133 if [ -z "${HEALTHCHECK_PING_URL:-}" ]; then
134 echo "HEALTHCHECK_PING_URL secret not set — no external page will fire."
135 exit 0
136 fi
137 curl -fsS --max-time 10 "${HEALTHCHECK_PING_URL%/}/fail" >/dev/null 2>&1 || true
138 echo "dead-man /fail pinged"
139
140 - name: Ping external dead-man (ok)
141 if: success()
142 env:
143 HEALTHCHECK_PING_URL: ${{ secrets.HEALTHCHECK_PING_URL }}
144 run: |
145 set -u
146 [ -z "${HEALTHCHECK_PING_URL:-}" ] && { echo "HEALTHCHECK_PING_URL unset — skipping"; exit 0; }
147 curl -fsS --max-time 10 "${HEALTHCHECK_PING_URL%/}" >/dev/null 2>&1 || true
148 echo "dead-man ok pinged"
149
121150 - name: Close the outage issue on recovery
122151 if: success()
123152 env:
Modifiedsrc/__tests__/workflow-runner-concurrency.test.ts+21−0View fileUnifiedSplit
@@ -63,3 +63,24 @@ describe("the tick drains through a bounded pool", () => {
6363 expect(tick).toContain("Promise.race(inFlight)");
6464 });
6565});
66
67describe("shouldDeferForLoad — the self-DDoS guard", () => {
68 test("defers above threshold, claims below", async () => {
69 const { shouldDeferForLoad } = await import("../lib/workflow-runner");
70 expect(shouldDeferForLoad({}, 9.0, 4)).toBe(true); // 9 > 8 (2x4)
71 expect(shouldDeferForLoad({}, 7.9, 4)).toBe(false);
72 });
73
74 test("WORKFLOW_LOAD_DEFER overrides the derived threshold", async () => {
75 const { shouldDeferForLoad } = await import("../lib/workflow-runner");
76 expect(shouldDeferForLoad({ WORKFLOW_LOAD_DEFER: "3" }, 3.5, 4)).toBe(true);
77 expect(shouldDeferForLoad({ WORKFLOW_LOAD_DEFER: "12" }, 9.0, 4)).toBe(false);
78 // Garbage falls back to the derived default.
79 expect(shouldDeferForLoad({ WORKFLOW_LOAD_DEFER: "nope" }, 9.0, 4)).toBe(true);
80 });
81
82 test("Windows loadavg of 0 never defers", async () => {
83 const { shouldDeferForLoad } = await import("../lib/workflow-runner");
84 expect(shouldDeferForLoad({}, 0, 4)).toBe(false);
85 });
86});
Modifiedsrc/lib/workflow-runner.ts+49−2View fileUnifiedSplit
@@ -17,7 +17,7 @@
1717 */
1818import { and, asc, eq, lt, sql } from "drizzle-orm";
1919import { mkdtemp, rm } from "fs/promises";
20import { tmpdir } from "os";
20import { cpus, loadavg, tmpdir } from "os";
2121import { join } from "path";
2222import { config } from "./config";
2323import { db } from "../db";
@@ -540,7 +540,15 @@ async function runStep(
540540 let escalateTimer: ReturnType<typeof setTimeout> | null = null;
541541
542542 try {
543 proc = Bun.spawn(["bash", "-c", run], {
543 // `nice -n 10` on POSIX hosts: even a running CI step yields the CPU to
544 // the app serving traffic beside it (INCIDENT 2026-08-22 — CI starved
545 // the platform's own request handlers). Windows dev has no `nice`; the
546 // step runs unniced there, which only ever affects a dev machine.
547 const stepCmd =
548 process.platform === "win32"
549 ? ["bash", "-c", run]
550 : ["nice", "-n", "10", "bash", "-c", run];
551 proc = Bun.spawn(stepCmd, {
544552 cwd: checkoutDir,
545553 stdout: "pipe",
546554 stderr: "pipe",
@@ -1082,11 +1090,50 @@ async function loadRun(runId: string) {
10821090 return row || null;
10831091}
10841092
1093// ---------------------------------------------------------------------------
1094// Load-aware claim guard — the structural fix for INCIDENT 2026-08-22.
1095// ---------------------------------------------------------------------------
1096
1097/**
1098 * The runner shares CPU with the app it lives inside (and, on the prod box,
1099 * with a dozen unfenced co-tenants). When 1-minute load is already above
1100 * threshold, claiming another run makes the platform strangle itself — the
1101 * incident shape was load ~120 on 4 cores with every HTTP connect timing
1102 * out. Deferring costs nothing: the row stays queued and the next tick
1103 * (2s) re-checks. Threshold defaults to 2× core count; override with
1104 * WORKFLOW_LOAD_DEFER (absolute 1-min load). loadavg is [0,0,0] on
1105 * Windows, so dev boxes never defer.
1106 */
1107export function shouldDeferForLoad(
1108 env: Record<string, string | undefined> = process.env,
1109 loadavg1: number = loadavg()[0],
1110 cores: number = cpus().length || 1
1111): boolean {
1112 const raw = Number(env.WORKFLOW_LOAD_DEFER);
1113 const threshold =
1114 Number.isFinite(raw) && raw > 0 ? raw : Math.max(2, cores * 2);
1115 return loadavg1 > threshold;
1116}
1117
1118let lastLoadDeferLogAt = 0;
1119
10851120// ---------------------------------------------------------------------------
10861121// Public: drainOneRun — pick + execute the oldest queued row.
10871122// ---------------------------------------------------------------------------
10881123
10891124export async function drainOneRun(): Promise<boolean> {
1125 if (shouldDeferForLoad()) {
1126 // Hourly log, not per-2s-tick — the condition can persist for minutes
1127 // during a legitimate heavy run and the queue view already shows depth.
1128 const now = Date.now();
1129 if (now - lastLoadDeferLogAt > 60 * 60_000) {
1130 lastLoadDeferLogAt = now;
1131 console.warn(
1132 `[workflow-runner] deferring CI claims — 1-min load ${loadavg()[0].toFixed(1)} above threshold; queued runs wait (repeats hourly while true)`
1133 );
1134 }
1135 return false;
1136 }
10901137 let candidateId: string | null = null;
10911138 try {
10921139 const [row] = await db
10931140
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts