feat(ci): boot-time sweep requeues runs orphaned by the deploy restart #5468
2 changed files+166−0
Modifiedsrc/__tests__/workflow-run-abandonment.test.ts+47−0View fileUnifiedSplit
@@ -101,6 +101,53 @@ describe("runs reach a terminal status", () => {
101101 expect(fn).toMatch(/\.limit\(\d+\)/);
102102 });
103103
104 test("the restart sweep runs once, before the reaper, before draining", async () => {
105 // A run left `running` by the process this deploy replaced is dead with
106 // certainty — the boot sweep finalises it immediately instead of after
107 // the reaper's hour, and retries recent victims. Order matters for the
108 // same reason as the reaper: it must fire before the tick starts
109 // draining. Run #257 (2026-08-08) is why this exists: the verify-deploy
110 // workflow is killed BY the deploy it verifies on every push to main,
111 // so without a retry it could never go green.
112 const src = await Bun.file("src/lib/workflow-runner.ts").text();
113 const code = src
114 .split("\n")
115 .filter((l) => !l.trimStart().startsWith("*") && !l.trimStart().startsWith("//"))
116 .join("\n");
117 const tick = code.slice(code.indexOf("const tick = async"));
118 const sweep = tick.indexOf("requeueRunsOrphanedByRestart(");
119 const reap = tick.indexOf("reapStuckRuns()");
120 const drain = tick.indexOf("drainOneRun()");
121 expect(sweep).toBeGreaterThan(-1);
122 expect(reap).toBeGreaterThan(sweep);
123 expect(drain).toBeGreaterThan(reap);
124 // One-shot: guarded by a flag that is set BEFORE the await, so a
125 // reentrant tick can never run the sweep twice.
126 const guard = tick.indexOf("restartSweepDone = true");
127 expect(guard).toBeGreaterThan(-1);
128 expect(guard).toBeLessThan(sweep);
129 });
130
131 test("the restart sweep is bounded, guarded against retry loops, and never throws", async () => {
132 const src = await Bun.file("src/lib/workflow-runner.ts").text();
133 const fn = src.slice(
134 src.indexOf("export async function requeueRunsOrphanedByRestart"),
135 src.indexOf("async function markRunFailed")
136 );
137 // Bounded per sweep — same Neon-hot-path argument as the reaper.
138 expect(fn).toMatch(/\.limit\(\d+\)/);
139 // One retry per (workflow, commit): a workflow whose execution restarts
140 // its own container must not requeue itself on every boot, forever.
141 expect(fn).toContain('"runner_restarted"');
142 expect(fn).toContain("alreadyRetried");
143 // Stale orphans are finalised but not retried — their commit has been
144 // superseded and the drain is serial.
145 expect(fn).toContain("RESTART_REQUEUE_WINDOW_MS");
146 // Never throws into the tick — a failed sweep must not stop draining.
147 expect(fn).toContain("catch");
148 expect(fn).toContain("return 0;");
149 });
150
104151 test("the reaper never throws into the tick", async () => {
105152 // It is the first thing the tick does. If it threw, the catch below
106153 // would skip draining entirely and the runner would stop executing
Modifiedsrc/lib/workflow-runner.ts+119−0View fileUnifiedSplit
@@ -327,6 +327,114 @@ export async function reapStuckRuns(now: Date = new Date()): Promise<number> {
327327 }
328328}
329329
330/**
331 * Only orphans killed this recently get a retry. An orphan from hours ago
332 * is stale news — its commit has been superseded, and requeueing it would
333 * make the serial drain spend minutes re-running work nobody is waiting
334 * for, ahead of runs someone is.
335 */
336export const RESTART_REQUEUE_WINDOW_MS = 15 * 60_000;
337
338/**
339 * Boot-time sweep: finalise and retry runs the previous process died holding.
340 *
341 * Runs execute in-process and this box deploys by recreating the container,
342 * so a row still `running` from before this process booted is provably dead
343 * — no cutoff heuristic needed, unlike reapStuckRuns(). The routine victim
344 * is the verify-deploy workflow, which is killed BY the very deploy it
345 * verifies on every push to main; under reaping alone it could never go
346 * green (run #257, 2026-08-08, was the proof).
347 *
348 * So: fail each orphan honestly (`runner_restarted`), and for recent ones
349 * enqueue a single fresh attempt, which now runs against the post-deploy
350 * world. At most one retry per (workflow, commit) — a workflow whose
351 * execution restarts its own container would otherwise requeue itself on
352 * every boot, forever.
353 *
354 * Single-instance assumption, deliberate: with two app processes sharing
355 * the database this would kill the other instance's live runs. This
356 * deployment is one container by design (docker-compose.standalone.yml).
357 */
358export async function requeueRunsOrphanedByRestart(
359 bootTime: Date
360): Promise<number> {
361 try {
362 const orphans = await db
363 .select({
364 id: workflowRuns.id,
365 workflowId: workflowRuns.workflowId,
366 repositoryId: workflowRuns.repositoryId,
367 event: workflowRuns.event,
368 ref: workflowRuns.ref,
369 commitSha: workflowRuns.commitSha,
370 triggeredBy: workflowRuns.triggeredBy,
371 startedAt: workflowRuns.startedAt,
372 })
373 .from(workflowRuns)
374 .where(
375 and(
376 eq(workflowRuns.status, "running"),
377 lt(workflowRuns.startedAt, bootTime)
378 )
379 )
380 .limit(50);
381
382 let requeued = 0;
383 for (const run of orphans) {
384 // One retry per (workflow, commit): a prior runner_restarted row for
385 // this pair means this orphan IS the retry — fail it, don't chain.
386 let alreadyRetried = false;
387 if (run.commitSha) {
388 try {
389 const [prior] = await db
390 .select({ id: workflowRuns.id })
391 .from(workflowRuns)
392 .where(
393 and(
394 eq(workflowRuns.workflowId, run.workflowId),
395 eq(workflowRuns.commitSha, run.commitSha),
396 eq(workflowRuns.conclusion, "runner_restarted")
397 )
398 )
399 .limit(1);
400 alreadyRetried = Boolean(prior);
401 } catch {
402 // If the guard can't be read, assume retried — the safe direction
403 // is a missing retry, never a retry loop.
404 alreadyRetried = true;
405 }
406 }
407
408 await markRunFailed(run.id, "runner_restarted");
409
410 const tooOld =
411 !run.startedAt ||
412 bootTime.getTime() - run.startedAt.getTime() >
413 RESTART_REQUEUE_WINDOW_MS;
414 if (alreadyRetried || tooOld) continue;
415
416 const newId = await enqueueRun({
417 workflowId: run.workflowId,
418 repositoryId: run.repositoryId,
419 event: run.event,
420 ref: run.ref,
421 commitSha: run.commitSha,
422 triggeredBy: run.triggeredBy,
423 });
424 if (newId) requeued += 1;
425 }
426 if (orphans.length > 0) {
427 console.error(
428 `[workflow-runner] restart sweep: ${orphans.length} orphaned run(s) marked runner_restarted, ${requeued} requeued`
429 );
430 }
431 return requeued;
432 } catch (err) {
433 console.error("[workflow-runner] requeueRunsOrphanedByRestart:", err);
434 return 0;
435 }
436}
437
330438async function markRunFailed(
331439 runId: string,
332440 conclusion: string
@@ -1085,8 +1193,10 @@ export async function enqueueRun(opts: {
10851193
10861194export function startWorker(opts?: { intervalMs?: number }): () => void {
10871195 const intervalMs = opts?.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
1196 const bootTime = new Date();
10881197 let stopped = false;
10891198 let active = false;
1199 let restartSweepDone = false;
10901200 // A run can legitimately take many minutes, so there is no blanket
10911201 // ceiling here — but a tick that never settles wedges this worker
10921202 // permanently and silently. 45 minutes is well past any real run.
@@ -1113,6 +1223,15 @@ export function startWorker(opts?: { intervalMs?: number }): () => void {
11131223 // recreates the container on every deploy, roughly once a minute when
11141224 // commits are landing. Cheap: one indexed lookup per tick, and it
11151225 // touches nothing younger than an hour.
1226 //
1227 // The restart sweep runs once, first: runs left `running` by the
1228 // process this deploy replaced are dead with certainty (they executed
1229 // in that process), so they can be finalised — and recent ones retried
1230 // — immediately instead of waiting out the hour-long reaper cutoff.
1231 if (!restartSweepDone) {
1232 restartSweepDone = true;
1233 await requeueRunsOrphanedByRestart(bootTime);
1234 }
11161235 await reapStuckRuns();
11171236
11181237 // Drain as many runs as we can in one tick (serial). If there's
11191238
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts