fix(ci): moving checkouts to a shared volume made them leak forever #5605
2 changed files+196−2
Addedsrc/__tests__/ci-workspace-sweep.test.ts+118−0View fileUnifiedSplit
@@ -0,0 +1,118 @@
1/**
2 * Bounding the ci-work volume.
3 *
4 * Checkouts used to be mkdtemp'd inside the app container, so a leftover
5 * directory vanished with the next container restart — roughly hourly on this
6 * box. Moving them to the shared volume so a sibling runner could see them
7 * (2026-08-31) made them PERSISTENT, and turned a self-clearing mess into an
8 * unbounded one.
9 *
10 * Measured hours after that change: six orphaned checkouts, 1.2 GB, on a
11 * 150 GB disk shared with five other tenants and already 64% full. Each is a
12 * clone plus node_modules. A full disk here is an outage for every tenant on
13 * the box.
14 *
15 * These run against a real temp directory rather than a mocked fs, because
16 * the thing being asserted is what happens to files.
17 */
18
19import { afterEach, beforeEach, describe, expect, test } from "bun:test";
20import { mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync, existsSync, readdirSync } from "fs";
21import { tmpdir } from "os";
22import { join } from "path";
23import { sweepStaleWorkspaces, WORKSPACE_MAX_AGE_MS } from "../lib/workflow-runner";
24
25let root = "";
26
27/** A workspace directory with a checkout in it, aged by `ageMs`. */
28function makeWorkspace(name: string, ageMs: number): string {
29 const dir = join(root, name);
30 mkdirSync(join(dir, "checkout"), { recursive: true });
31 writeFileSync(join(dir, "checkout", "file.txt"), "x");
32 const when = new Date(Date.now() - ageMs);
33 utimesSync(dir, when, when);
34 return dir;
35}
36
37beforeEach(() => {
38 root = mkdtempSync(join(tmpdir(), "sweep-test-"));
39});
40
41afterEach(() => {
42 rmSync(root, { recursive: true, force: true });
43});
44
45describe("orphaned checkouts are removed, live ones are not", () => {
46 test("a checkout older than the age limit is deleted", () => {
47 const old = makeWorkspace("gluecron-run-OLD001", WORKSPACE_MAX_AGE_MS + 60_000);
48 return sweepStaleWorkspaces(Date.now(), root).then((n) => {
49 expect(n).toBe(1);
50 expect(existsSync(old)).toBe(false);
51 });
52 });
53
54 test("a recent checkout is left completely alone", async () => {
55 // The failure that would matter: deleting the working directory of a job
56 // that is still running. Worse than keeping a dead one for an extra hour,
57 // which is why the limit is two hours against a one-hour stuck-run reap.
58 const fresh = makeWorkspace("gluecron-run-LIVE01", 60_000);
59 const n = await sweepStaleWorkspaces(Date.now(), root);
60 expect(n).toBe(0);
61 expect(existsSync(join(fresh, "checkout", "file.txt"))).toBe(true);
62 });
63
64 test("a checkout exactly at the limit is kept, not deleted", async () => {
65 // Boundary in the safe direction.
66 makeWorkspace("gluecron-run-EDGE01", WORKSPACE_MAX_AGE_MS);
67 expect(await sweepStaleWorkspaces(Date.now(), root)).toBe(0);
68 });
69});
70
71describe("the sweep only ever touches its own directories", () => {
72 test("anything not named gluecron-run-* survives, however old", async () => {
73 // The volume is shared. Deleting a directory this code did not create is
74 // not a cleanup, it is data loss in someone else's service.
75 const foreign = join(root, "someone-elses-data");
76 mkdirSync(foreign, { recursive: true });
77 writeFileSync(join(foreign, "important.txt"), "keep me");
78 const when = new Date(Date.now() - WORKSPACE_MAX_AGE_MS * 10);
79 utimesSync(foreign, when, when);
80
81 const n = await sweepStaleWorkspaces(Date.now(), root);
82 expect(n).toBe(0);
83 expect(existsSync(join(foreign, "important.txt"))).toBe(true);
84 });
85
86 test("a stray FILE with the right prefix is not deleted", async () => {
87 // Only directories. A file named like a workspace is not one.
88 const stray = join(root, "gluecron-run-notadir");
89 writeFileSync(stray, "x");
90 const when = new Date(Date.now() - WORKSPACE_MAX_AGE_MS * 5);
91 utimesSync(stray, when, when);
92 expect(await sweepStaleWorkspaces(Date.now(), root)).toBe(0);
93 expect(existsSync(stray)).toBe(true);
94 });
95});
96
97describe("the sweep cannot break the tick it runs on", () => {
98 test("no configured root is a no-op, not an error", async () => {
99 // Dev machines and single-container self-hosters have no shared volume.
100 expect(await sweepStaleWorkspaces(Date.now(), "")).toBe(0);
101 });
102
103 test("a missing root directory is survived", async () => {
104 expect(await sweepStaleWorkspaces(Date.now(), join(root, "does-not-exist"))).toBe(0);
105 });
106
107 test("mixed content sweeps what it should and keeps the rest", async () => {
108 makeWorkspace("gluecron-run-A", WORKSPACE_MAX_AGE_MS * 3);
109 makeWorkspace("gluecron-run-B", WORKSPACE_MAX_AGE_MS * 2);
110 makeWorkspace("gluecron-run-C", 1_000);
111 mkdirSync(join(root, "keep-me"), { recursive: true });
112
113 const n = await sweepStaleWorkspaces(Date.now(), root);
114 expect(n).toBe(2);
115 const left = readdirSync(root).sort();
116 expect(left).toEqual(["gluecron-run-C", "keep-me"]);
117 });
118});
Modifiedsrc/lib/workflow-runner.ts+78−2View fileUnifiedSplit
@@ -16,7 +16,7 @@
1616 * - startWorker({ interval }) — background poll loop (returns stop fn)
1717 */
1818import { and, asc, eq, lt, sql } from "drizzle-orm";
19import { mkdir, mkdtemp, rm } from "fs/promises";
19import { mkdir, mkdtemp, readdir, rm, stat } from "fs/promises";
2020import {
2121 execViaRunner,
2222 liveRunIds,
@@ -313,6 +313,74 @@ export const STUCK_RUN_MS = 60 * 60_000;
313313 * older than STUCK_RUN_MS, so a run this worker is actively executing is
314314 * out of range by an order of magnitude.
315315 */
316/**
317 * How long a checkout may sit before it is presumed orphaned.
318 *
319 * Two hours, against `reapStuckRuns` marking anything still running after ONE
320 * hour as abandoned. Nothing legitimately holds a checkout past that, so the
321 * extra hour is pure margin — deleting a live job's working directory would
322 * be a far worse bug than keeping a dead one for an extra hour.
323 */
324export const WORKSPACE_MAX_AGE_MS = 2 * 60 * 60_000;
325
326/**
327 * Delete checkout directories left behind by runs that were killed.
328 *
329 * WHY THIS IS NEEDED NOW AND WAS NOT BEFORE. Checkouts used to be mkdtemp'd
330 * inside the app container, so a leftover directory vanished with the next
331 * container restart — which on this box is roughly hourly. Moving them to the
332 * shared `ci-work` volume so a sibling runner could see them (2026-08-31) made
333 * them PERSISTENT, and turned a self-clearing mess into an unbounded one.
334 *
335 * Measured hours after that change: six orphaned checkouts, 1.2 GB, on a
336 * 150 GB disk shared with five other tenants that was already 64% full. Each
337 * is a full clone plus node_modules. At this instance's run rate that is
338 * gigabytes a day, and a full disk here is an outage for every tenant on the
339 * box — the same failure the broker's container cleanup exists to prevent.
340 *
341 * The dominant cause cannot be fixed in-process: the routine victim is a run
342 * killed by the very deploy that recreates the container, and no `finally`
343 * survives SIGKILL. So this is a sweeper, on the same schedule as the other
344 * two, and it is the thing that actually bounds the growth.
345 *
346 * Never throws. A sweeper that can break the tick it runs on is worse than
347 * the disk it was protecting.
348 */
349export async function sweepStaleWorkspaces(
350 now: number = Date.now(),
351 root: string = process.env["GLUECRON_CI_WORKDIR"] || ""
352): Promise<number> {
353 if (!root) return 0;
354 let removed = 0;
355 try {
356 const entries = await readdir(root);
357 for (const name of entries) {
358 // Only directories this code created. Anything else in the volume was
359 // put there by someone else and is not ours to delete.
360 if (!name.startsWith("gluecron-run-")) continue;
361 const full = join(root, name);
362 try {
363 const info = await stat(full);
364 if (!info.isDirectory()) continue;
365 if (now - info.mtimeMs <= WORKSPACE_MAX_AGE_MS) continue;
366 await rm(full, { recursive: true, force: true });
367 removed += 1;
368 } catch {
369 // One unreadable entry must not stop the sweep; the next tick retries.
370 continue;
371 }
372 }
373 if (removed > 0) {
374 console.error(
375 `[workflow-runner] swept ${removed} orphaned checkout(s) from ${root}`
376 );
377 }
378 } catch (err) {
379 console.error("[workflow-runner] sweepStaleWorkspaces:", err);
380 }
381 return removed;
382}
383
316384export async function reapStuckRuns(now: Date = new Date()): Promise<number> {
317385 const cutoff = new Date(now.getTime() - STUCK_RUN_MS);
318386 try {
@@ -1407,7 +1475,12 @@ export async function executeRun(runId: string): Promise<void> {
14071475 }
14081476 }
14091477
1410 // Cleanup always runs.
1478 // Cleanup on the ordinary path. It is NOT guaranteed — this line is not in
1479 // a finally, and more importantly a SIGKILL cannot be caught at all, so a
1480 // run killed by the deploy that is recreating this container leaves its
1481 // checkout behind. sweepStaleWorkspaces() below is what actually bounds
1482 // that; this is only the fast path. The comment here used to say "cleanup
1483 // always runs", which was a claim the code did not support.
14111484 await rm(tmpRoot, { recursive: true, force: true }).catch((err) => {
14121485 console.error("[workflow-runner] tmpdir cleanup:", err);
14131486 });
@@ -1648,6 +1721,9 @@ export function startWorker(opts?: {
16481721 await requeueRunsOrphanedByRestart(bootTime);
16491722 }
16501723 await reapStuckRuns();
1724 // Bounds the ci-work volume. See sweepStaleWorkspaces() for why the
1725 // in-run cleanup is not sufficient on its own.
1726 await sweepStaleWorkspaces();
16511727
16521728 // Drain the queue through a bounded pool (WORKFLOW_CONCURRENCY,
16531729 // default 3; was strictly serial until 2026-08-22 — one slow run
16541730
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts