Code Issues Pull Requests Actions Security InsightsSettings

feat(ci): execute workflow steps in a sibling container, not the app #5595

MergedXSccantynz wants to mergefeat/ci-sidecar-runner→mainopened 1d ago

8 changed files+957−16

Modified.env.example+33−0View fileUnified Split

@@ -468,3 +468,36 @@ EMAIL_HTTP_TOKEN=

468468#

469469# Production: WORKFLOW_EXEC_ALLOWLIST=ccantynz

470470WORKFLOW_EXEC_ALLOWLIST=

471

472# ── CI runner sidecar ──────────────────────────────────────────────────────

473#

474# Moves workflow `run:` steps out of the app container and into a sibling

475# container that has no .env, no /data/repos and no database. See

476# docs/CI-ISOLATION-NEXT-INCREMENT.md and src/lib/runner-rpc.ts.

477#

478# Both empty by default: the runner comes up alongside and is verified before

479# anything routes to it. The app treats a URL without a token as no runner at

480# all, so a half-configured box keeps running CI the old way rather than

481# turning every build red at once.

482#

483# To enable, set BOTH and redeploy:

484# RUNNER_URL=http://runner:3001

485# RUNNER_TOKEN=$(openssl rand -hex 32)

486# The token must match on both services; it is the only thing standing

487# between the container network and arbitrary shell execution.

488RUNNER_URL=

489RUNNER_TOKEN=

490

491# Where checkouts are written so both containers can see them. Set to

492# /ci-work by docker-compose.standalone.yml, which mounts the same volume

493# there on the app and the runner. Unset = the ordinary temp dir (dev).

494GLUECRON_CI_WORKDIR=

495

496# The runner's own bounds. mem_limit is what stops a build from being

497# constrained by the running site's memory budget — measured, not guessed:

498# tsc --noEmit peaks at 3,349 MB on this repo (2026-08-29), and a vitest gate

499# forks a node worker per core on top of that.

500RUNNER_MEM_LIMIT=5g

501RUNNER_CPUS=2.0

502# Port the agent listens on inside its container. Never published.

503RUNNER_PORT=3001

Modifieddocker-compose.standalone.yml+72−0View fileUnified Split

@@ -84,10 +84,29 @@ services:

8484 - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}

8585 - GOOGLE_OAUTH_AUTO_CREATE=${GOOGLE_OAUTH_AUTO_CREATE:-}

8686 - GOOGLE_OAUTH_ALLOWED_DOMAINS=${GOOGLE_OAUTH_ALLOWED_DOMAINS:-}

87 # CI execution moves to the sibling `runner` service below — but only

88 # once BOTH of these are set in .env, deliberately, by an operator who

89 # has confirmed the runner is healthy:

90 #

91 # RUNNER_URL=http://runner:3001

92 # RUNNER_TOKEN=<openssl rand -hex 32>

93 #

94 # Defaulting them empty means this deploy changes nothing about how CI

95 # executes; the runner comes up alongside and can be verified first.

96 # The app treats a URL without a token as no runner at all (see

97 # runnerUrl()), so a half-set .env falls back to the working path

98 # instead of turning every build on the platform red at once.

99 - RUNNER_URL=${RUNNER_URL:-}

100 - RUNNER_TOKEN=${RUNNER_TOKEN:-}

101 - GLUECRON_CI_WORKDIR=/ci-work

87102 expose:

88103 - "3000"

89104 volumes:

90105 - git-repos:/data/repos

106 # The ONLY filesystem the runner shares with the app. Checkouts are

107 # written here by the app (which has the repos and the credentials) and

108 # read by the runner (which has neither).

109 - ci-work:/ci-work

91110 # NO depends_on: postgres. Production's database is Neon (DATABASE_URL);

92111 # the local `postgres` service below is unused dead weight (see

93112 # CLAUDE.md). Gating the app on that container's health meant a corrupt

@@ -160,6 +179,58 @@ services:

160179 retries: 3

161180 start_period: 60s

162181

182 # ── CI runner ──────────────────────────────────────────────────────────

183 # Same image, different entrypoint, and deliberately impoverished: no

184 # env_file, no /data/repos, no database. A workflow `run:` step executes

185 # HERE, so the three things that were true of running it inside the app are

186 # no longer true — see src/lib/runner-rpc.ts for the full reasoning.

187 #

188 # What it must NOT gain: `env_file: .env` (hands every platform secret to

189 # every step), a `git-repos` mount (hands every repository to every step),

190 # or a `ports:` mapping (the token is the only thing between the internet

191 # and arbitrary shell execution). If you find yourself adding any of them,

192 # the thing you want is a different design, not this service with one more

193 # line.

194 #

195 # NOT closed by this: two jobs in sequence share this container. That is why

196 # src/lib/workflow-exec-policy.ts still restricts who may run CI.

197 runner:

198 build: .

199 command: ["bun", "run", "src/runner-agent.ts"]

200 # No env_file. Everything the step needs is sent per-request by the app.

201 environment:

202 - RUNNER_PORT=3001

203 - RUNNER_TOKEN=${RUNNER_TOKEN:-}

204 - NODE_ENV=production

205 expose:

206 - "3001"

207 volumes:

208 - ci-work:/ci-work

209 restart: unless-stopped

210 logging: *default-logging

211 # The build's memory now lives here instead of bounding the running site.

212 # Measured 2026-08-29: tsc --noEmit peaks at 3,349 MB on this repo, and a

213 # vitest gate forks one node worker per core on top of that. 5g, and the

214 # number is a measurement — the two previous ceilings on the app service

215 # were guesses and both SIGKILLed CI.

216 mem_limit: ${RUNNER_MEM_LIMIT:-5g}

217 memswap_limit: ${RUNNER_MEM_LIMIT:-5g}

218 # A build may use the box's CPU, but never all of it: the site is served

219 # by the container beside this one.

220 cpus: ${RUNNER_CPUS:-2.0}

221 pids_limit: 512

222 security_opt:

223 - no-new-privileges:true

224 ulimits:

225 core: 0

226 stop_grace_period: 30s

227 healthcheck:

228 test: ["CMD", "wget", "-qO-", "http://localhost:3001/healthz"]

229 interval: 30s

230 timeout: 10s

231 retries: 3

232 start_period: 30s

233

163234 caddy:

164235 image: caddy:2-alpine

165236 restart: unless-stopped

@@ -190,5 +261,6 @@ services:

190261volumes:

191262 pgdata:

192263 git-repos:

264 ci-work:

193265 caddy-data:

194266 caddy-config:

Addeddocs/CI-ISOLATION-NEXT-INCREMENT.md+112−0View fileUnified Split

@@ -0,0 +1,112 @@

1# CI isolation — the next increment

3Follow-up to `AUDIT-CI-RUNNER-ISOLATION.md`. That document is still the

4authority on the target architecture and on what has been ruled out; this one

5records the **smallest increment that materially reduces risk**, designed

62026-08-31 against commit `6946606`, and the traps found while designing it.

8## Status

10**Built 2026-08-31 and merged; OFF in production until an operator enables it.**

12`RUNNER_URL` and `RUNNER_TOKEN` are both empty by default, so the runner

13container comes up alongside the app and nothing routes to it. The app treats a

14URL without a token as no runner at all, so a half-filled `.env` keeps CI

15working the old way instead of turning every build on the platform red at once.

17To turn it on, on the box: set both in `/opt/gluecron/.env`

18(`RUNNER_TOKEN=$(openssl rand -hex 32)`, `RUNNER_URL=http://runner:3001`), then

19let the update timer redeploy. To turn it off, clear `RUNNER_URL`; execution

20falls back to the in-process path with no code change.

22Verification is the "Done when" section at the bottom — run it inside a real

23step before trusting it, not against the compose file.

25## The move: a sibling runner container that receives the CHECKOUT

27Not the git volume. Not the docker socket. A second container from the same

28image, different `command:`, **no `env_file`, no `/data/repos` mount**, sharing

29one `ci-work` volume with the app. The app clones into `/ci-work/<runid>/checkout`

30— it already has the code and the access — and the runner only ever sees that

31directory.

33### What it closes, in about a day

35| Risk | Today | After |

36|---|---|---|

37| The app's memory limit bounds every build | A 2 GiB app cap SIGKILLed `tsc` and took CI down platform-wide (2026-08-30) | Runner has its own cgroup; the app's `mem_limit` drops back to app-sized |

38| A step can read the platform's secrets | Shares a PID namespace with PID 1, which holds the whole `.env` | The runner's PID 1 is the runner agent, whose env has no secrets — `/proc/1/environ` becomes worthless |

39| A step can read and rewrite every repository on the instance | `/data/repos` mounted read-write | **No filesystem path to any repository at all** |

40| CI starves the app of CPU | `nice -n 10` only | `cpus:` on the runner service |

42That third row is the sentence in `src/lib/workflow-exec-policy.ts` becoming

43false, which is the point of the whole exercise.

45### What it does NOT close, and must be said when reporting it

47**Job-to-job contamination.** One long-lived runner has no per-tenant boundary,

48which is exactly the objection the audit raises to runner pools. This increment

49is what lets CI be turned on for the FIRST customer; per-job ephemeral

50containers via the broker are what let it be turned on for the tenth.

52So `workflow-exec-policy.ts` **stays** after this lands, despite its own header

53saying to delete it when isolation arrives. Relax it, and rewrite the header to

54say what is then true. Delete it when the broker lands.

56Also untouched: egress/exfiltration, kernel escape, disk quota on `ci-work`.

58## The trap that would have bitten

60`requeueRunsOrphanedByRestart` (`src/lib/workflow-runner.ts`) assumes the

61process executing a run dies when the app dies. With a sibling runner **it does

62not**: an app redeploy leaves the step executing in the runner container while

63the app marks the run `runner_restarted` and re-enqueues it — two concurrent

64executions of the same job against the same checkout.

66Before `markRunFailed(run.id, "runner_restarted")`, ask the runner agent what it

67is currently executing (`GET /status`) and skip those run ids. This edit is not

68optional and it is not obvious from reading either file alone.

70`reapStuckRuns` has the same shape at hour scale, and wants the same answer.

72## Files

741. **`docker-compose.standalone.yml`** — new `runner` service (same `build: .`,

75 `command: ["bun","run","src/runner-agent.ts"]`, no `env_file`, `expose` only,

76 `read_only`, `tmpfs /tmp`, `cap_drop: [ALL]`, `no-new-privileges`,

77 `pids_limit`, its own `mem_limit`), plus a `ci-work` volume on both services.

78 Then **drop the app's `mem_limit` back to app-sized** — the comment there

79 already says raising it was a stopgap and that the real fix is this.

802. **`src/runner-agent.ts`** (new, ~150 lines) — one Bun HTTP server, bearer

81 token, `POST /exec` whose body is today's `runStep` internals lifted

82 verbatim, plus `GET /status` for the trap above. **No DB import** — that is

83 what keeps `DATABASE_URL` out of the process.

843. **`src/lib/workflow-runner.ts`** — three edits: `runStep` posts to

85 `RUNNER_URL` when set and otherwise falls through to today's spawn (both

86 paths live side by side, so no flag day and Windows dev keeps working);

87 `cloneAt` mkdtemps under `GLUECRON_CI_WORKDIR`; the requeue guard.

884. **`.env.example`** — `RUNNER_URL`, `RUNNER_TOKEN`, `RUNNER_MEM_LIMIT`,

89 `GLUECRON_CI_WORKDIR`. The env-drift rule will fail the build otherwise, as

90 it has three times this week.

92## One correction to the audit

94It proposes `--memory 1.5g` per job and flags the figure as a proposal rather

95than a measurement. It is too low. Measured 2026-08-30: `tsc --noEmit` alone

96peaks at **3,349 MB** on this repo. Budget **≥4 GB** for a job container that

97has to run this repo's own typecheck, or the first thing the new isolation does

98is reproduce the outage it was built to prevent.

100## Done when

101

102`bunx tsc --noEmit` and `bash scripts/ci-tests.sh` complete inside the runner

103container while the app sits under an app-sized `mem_limit`, and inside a live

104step:

105

106```

107tr '\0' '\n' < /proc/1/environ # no DATABASE_URL

108ls /data/repos # No such file or directory

109```

110

111Those two commands failing to find anything is the deliverable. Everything else

112is how we got there.

Addedsrc/__tests__/ci-runner-sidecar.test.ts+179−0View fileUnified Split

@@ -0,0 +1,179 @@

1/**

2 * The properties that make the sibling runner worth having.

3 *

4 * Every one of these is destroyable by a single well-meaning line — an

5 * `env_file: .env` added so "the runner can read a config", a `git-repos`

6 * mount added so "the runner can clone", an `import { db }` added so "the

7 * runner can record its own result". Each of those is a reasonable-sounding

8 * edit that silently returns us to executing user shell scripts next to the

9 * platform's secrets, and none of them would fail a build without this file.

10 *

11 * So these tests assert ABSENCES. That is unusual and deliberate: the value

12 * of this design is entirely in what the runner does not have.

13 */

15import { describe, expect, test } from "bun:test";

16import { readFileSync } from "fs";

17import { resolve } from "path";

18import { execViaRunner, liveRunIds } from "../lib/runner-rpc";

20const root = resolve(import.meta.dir, "../..");

22/**

23 * Comments are stripped before every assertion below.

24 *

25 * Not a detail: the first version of this file failed against its own subject,

26 * because the compose block explains that it must never gain an `env_file` and

27 * the agent's header names `import { db }` as the line that would undo it. A

28 * test that reads prose is testing that we still describe the property, which

29 * is exactly the false green it exists to prevent — the configuration is what

30 * must be checked.

31 */

32function stripYamlComments(text: string): string {

33 return text

34 .split(/\r?\n/)

35 .filter((l) => !/^\s*#/.test(l))

36 .join("\n");

37}

39function stripTsComments(text: string): string {

40 return text

41 .replace(/\/\*[\s\S]*?\*\//g, "")

42 .split(/\r?\n/)

43 .filter((l) => !/^\s*\/\//.test(l))

44 .join("\n");

45}

47const compose = stripYamlComments(

48 readFileSync(resolve(root, "docker-compose.standalone.yml"), "utf8")

49);

50const agentSrc = stripTsComments(

51 readFileSync(resolve(root, "src/runner-agent.ts"), "utf8")

52);

54/** The `runner:` service block, from its key to the next top-level service. */

55function runnerBlock(): string {

56 const lines = compose.split(/\r?\n/);

57 const start = lines.findIndex((l) => /^ {2}runner:\s*$/.test(l));

58 expect(start).toBeGreaterThan(-1);

59 const rest = lines.slice(start + 1);

60 const end = rest.findIndex((l) => /^ {2}\S/.test(l));

61 return rest.slice(0, end === -1 ? rest.length : end).join("\n");

62}

64describe("the runner container holds nothing worth stealing", () => {

65 test("no env_file — the platform's secrets never enter the process", () => {

66 // This is the /proc/1/environ hole. The app container passes the whole

67 // .env to PID 1; a step sharing that namespace can read all of it. The

68 // runner's PID 1 must have nothing to read.

69 expect(runnerBlock()).not.toMatch(/env_file/);

70 });

72 test("no repository mount — no filesystem path to anyone's code", () => {

73 const block = runnerBlock();

74 expect(block).not.toMatch(/git-repos/);

75 expect(block).not.toMatch(/\/data\/repos/);

76 });

78 test("no published port — the token is not the only thing exposed", () => {

79 // `expose:` is container-network only. `ports:` would publish arbitrary

80 // shell execution to the host's interfaces, guarded by one bearer token.

81 expect(runnerBlock()).not.toMatch(/^\s+ports:/m);

82 });

84 test("no database or config import — the absence is the security property", () => {

85 // A DB import pulls DATABASE_URL into this process and undoes the whole

86 // exercise. The agent may only import its own wire types.

87 const imports = [...agentSrc.matchAll(/from\s+"([^"]+)"/g)].map((m) => m[1]!);

88 const local = imports.filter((i) => i.startsWith("."));

89 expect(local).toEqual(["./lib/runner-rpc"]);

90 expect(agentSrc).not.toMatch(/DATABASE_URL|ANTHROPIC_API_KEY|WORKFLOW_SECRETS_KEY/);

91 });

93 test("app and runner share ci-work at the same path, and only that", () => {

94 // A checkout written at /ci-work/x by the app must be at /ci-work/x for

95 // the runner, or every step fails with a confusing ENOENT.

96 expect(compose).toMatch(/ci-work:\/ci-work/);

97 expect(compose.match(/ci-work:\/ci-work/g)!.length).toBe(2);

98 expect(compose).toMatch(/GLUECRON_CI_WORKDIR=\/ci-work/);

99 expect(compose).toMatch(/^volumes:[\s\S]*^ {2}ci-work:/m);

100 });

101});

102

103describe("a runner we cannot reach is our failure, not the user's", () => {

104 const req = { runId: "r1", run: "echo hi", cwd: "/ci-work/x", env: {}, timeoutMs: 1000 };

105

106 test("a non-2xx from the runner is a launchError, never exit 0", async () => {

107 const out = await execViaRunner(req, {

108 url: "http://runner:3001",

109 token: "t",

110 fetchImpl: (async () =>

111 new Response("boom", { status: 502 })) as unknown as typeof fetch,

112 });

113 expect(out.launchError).toContain("502");

114 // The dangerous shape: a platform failure that reads as a passing step.

115 expect(out.exitCode).toBeNull();

116 });

117

118 test("a network error names the runner rather than blaming the step", async () => {

119 const out = await execViaRunner(req, {

120 url: "http://runner:3001",

121 token: "t",

122 fetchImpl: (async () => {

123 throw new Error("ECONNREFUSED");

124 }) as unknown as typeof fetch,

125 });

126 expect(out.launchError).toMatch(/could not reach the runner/);

127 expect(out.launchError).toContain("ECONNREFUSED");

128 });

129

130 test("a real exit code passes through untouched", async () => {

131 const out = await execViaRunner(req, {

132 url: "http://runner:3001",

133 token: "t",

134 fetchImpl: (async () =>

135 new Response(

136 JSON.stringify({ exitCode: 1, stdout: "out", stderr: "err", timedOut: false })

137 )) as unknown as typeof fetch,

138 });

139 expect(out).toEqual({ exitCode: 1, stdout: "out", stderr: "err", timedOut: false });

140 });

141});

142

143describe("the restart sweep asks before it reaps", () => {

144 test("an unreachable runner reports nothing live", async () => {

145 // Fail-open in this direction on purpose: a runner that cannot answer is

146 // a runner whose runs really are dead, and requeueing them is correct.

147 // The opposite default would strand live runs as `running` forever on any

148 // transient status failure.

149 const live = await liveRunIds({

150 url: "http://runner:3001",

151 fetchImpl: (async () => {

152 throw new Error("down");

153 }) as unknown as typeof fetch,

154 });

155 expect(live.size).toBe(0);

156 });

157

158 test("run ids the runner is executing come back", async () => {

159 const live = await liveRunIds({

160 url: "http://runner:3001",

161 fetchImpl: (async () =>

162 new Response(JSON.stringify({ running: ["a", "b"] }))) as unknown as typeof fetch,

163 });

164 expect([...live].sort()).toEqual(["a", "b"]);

165 });

166

167 test("no RUNNER_URL means no remote call at all", async () => {

168 let called = false;

169 const live = await liveRunIds({

170 url: null,

171 fetchImpl: (async () => {

172 called = true;

173 return new Response("{}");

174 }) as unknown as typeof fetch,

175 });

176 expect(called).toBe(false);

177 expect(live.size).toBe(0);

178 });

179});

Addedsrc/lib/runner-rpc.ts+203−0View fileUnified Split

@@ -0,0 +1,203 @@

1/**

2 * The wire between the app and the CI runner container.

3 *

4 * WHY THIS EXISTS. Today a workflow `run:` step is a `Bun.spawn` inside the

5 * app container. That single fact is behind three separate problems, and all

6 * three are structural rather than bugs to be fixed in place:

7 *

8 * 1. The app's memory limit bounds every build. On 2026-08-29 a 2 GiB app

9 * cap SIGKILLed `tsc` and took CI down platform-wide; the fix was to

10 * raise the app's ceiling to 6g, which docker-compose.standalone.yml

11 * itself records as a stopgap.

12 * 2. A step shares a PID namespace with PID 1, whose environment is the

13 * whole `.env`. `buildRunnerEnv` carefully withholds those variables

14 * from the step's own env, and `tr '\0' '\n' < /proc/1/environ` hands

15 * them straight back.

16 * 3. `/data/repos` is mounted read-write, so a step has a filesystem path

17 * to every repository on the instance.

18 *

19 * Moving execution to a sibling container closes all three at once, and it

20 * closes them by ARCHITECTURE — the runner cannot leak `DATABASE_URL` because

21 * the runner's process never had it, and cannot reach a repository because no

22 * such path is mounted. That is a different class of guarantee from a

23 * denylist, which is only ever as good as its last review.

24 *

25 * WHAT THIS DOES NOT CLOSE. One long-lived runner has no boundary BETWEEN

26 * jobs: two runs in sequence share a filesystem and a process namespace. That

27 * is why `workflow-exec-policy.ts` stays after this lands. This increment is

28 * what makes CI safe to offer the first customer; per-job ephemeral containers

29 * are what make it safe to offer the tenth. Saying so is the whole point —

30 * an isolation story that overstates itself is worse than none, because it

31 * gets trusted.

32 *

33 * SECRETS ON THE WIRE. `${{ secrets.NAME }}` is substituted into the `run:`

34 * text before it is sent, so a step's own declared secrets DO cross this

35 * connection. They have to: the step needs them. This link is

36 * container-to-container on a private docker network and bearer-authenticated;

37 * it must never be exposed on a published port. The platform's secrets, by

38 * contrast, never cross it at all.

39 */

41export interface ExecRequest {

42 /** The run this step belongs to. Reported back by GET /status. */

43 runId: string;

44 /** The `run:` script, secrets already substituted. */

45 run: string;

46 /** Absolute path inside the shared ci-work volume. */

47 cwd: string;

48 /** The full environment for the step — the runner adds nothing of its own. */

49 env: Record<string, string>;

50 /** Hard timeout; the runner owns the kill, because it owns the process. */

51 timeoutMs: number;

52}

54/**

55 * What actually happened, with no interpretation applied.

56 *

57 * `launchError` is distinct from a non-zero exit on purpose: "your command

58 * failed" and "we could not start your command" are different facts, and

59 * collapsing them is how a platform failure gets reported as the user's bug —

60 * the exact thing docs/CI-ATTRIBUTION-GUARANTEE.md promises we will not do.

61 */

62export interface ExecOutcome {

63 exitCode: number | null;

64 stdout: string;

65 stderr: string;

66 timedOut: boolean;

67 launchError?: string;

68}

70export interface RunnerStatus {

71 /** Run ids the runner is executing a step for right now. */

72 running: string[];

73}

75/**

76 * The sibling runner, if one is genuinely usable. Null = spawn locally.

77 *

78 * A URL WITHOUT a token counts as absent, and that is the important part. The

79 * runner refuses every unauthenticated /exec, so a deploy that set RUNNER_URL

80 * but not RUNNER_TOKEN would route all CI to a container that answers 401 —

81 * every build on the platform red, at once, from an omission in .env. Falling

82 * back to the in-process path is worse isolation and a working platform,

83 * which is the right way round for a half-finished configuration.

84 */

85export function runnerUrl(): string | null {

86 const raw = process.env["RUNNER_URL"];

87 if (!raw || raw.trim().length === 0) return null;

88 if (runnerToken().length === 0) return null;

89 return raw.replace(/\/+$/, "");

90}

92export function runnerToken(): string {

93 return process.env["RUNNER_TOKEN"] ?? "";

94}

96/**

97 * Execute one step on the sibling runner.

98 *

99 * Never throws: every failure becomes a `launchError`, because a network

100 * hiccup between two of our own containers is our fault and must be reported

101 * as such, not as the user's step failing.

102 *

103 * The client-side deadline is deliberately LONGER than the step's own — the

104 * runner is expected to enforce the timeout and answer with `timedOut: true`.

105 * This one only catches a runner that has stopped answering at all, and its

106 * message says so, so the two cases never get confused in a log.

107 */

108export async function execViaRunner(

109 req: ExecRequest,

110 opts: { url?: string | null; token?: string; fetchImpl?: typeof fetch } = {}

111): Promise<ExecOutcome> {

112 const url = opts.url ?? runnerUrl();

113 if (!url) {

114 return {

115 exitCode: null,

116 stdout: "",

117 stderr: "",

118 timedOut: false,

119 launchError: "no RUNNER_URL configured",

120 };

121 }

122 const doFetch = opts.fetchImpl ?? fetch;

123 const token = opts.token ?? runnerToken();

124

125 const controller = new AbortController();

126 const guard = setTimeout(

127 () => controller.abort(),

128 req.timeoutMs + RUNNER_RESPONSE_SLACK_MS

129 );

130 try {

131 const res = await doFetch(`${url}/exec`, {

132 method: "POST",

133 headers: {

134 "content-type": "application/json",

135 authorization: `Bearer ${token}`,

136 },

137 body: JSON.stringify(req),

138 signal: controller.signal,

139 });

140 if (!res.ok) {

141 const detail = (await res.text().catch(() => "")).slice(0, 500);

142 return {

143 exitCode: null,

144 stdout: "",

145 stderr: "",

146 timedOut: false,

147 launchError: `runner returned ${res.status}${detail ? `: ${detail}` : ""}`,

148 };

149 }

150 const body = (await res.json()) as Partial<ExecOutcome>;

151 return {

152 exitCode: typeof body.exitCode === "number" ? body.exitCode : null,

153 stdout: typeof body.stdout === "string" ? body.stdout : "",

154 stderr: typeof body.stderr === "string" ? body.stderr : "",

155 timedOut: body.timedOut === true,

156 ...(typeof body.launchError === "string"

157 ? { launchError: body.launchError }

158 : {}),

159 };

160 } catch (err) {

161 const msg = (err as Error)?.name === "AbortError"

162 ? `runner did not answer within ${req.timeoutMs + RUNNER_RESPONSE_SLACK_MS}ms — the runner container may be down`

163 : `could not reach the runner: ${(err as Error).message}`;

164 return { exitCode: null, stdout: "", stderr: "", timedOut: false, launchError: msg };

165 } finally {

166 clearTimeout(guard);

167 }

168}

169

170/** Grace beyond the step's own timeout before we call the runner unreachable. */

171export const RUNNER_RESPONSE_SLACK_MS = 30_000;

172

173/**

174 * Which runs the sibling runner is executing right now.

175 *

176 * Used by the restart sweep. Returns an EMPTY set when the runner cannot be

177 * reached, which is the correct failure direction: an unreachable runner is a

178 * dead runner, its runs really are orphaned, and the old behaviour — fail them

179 * and requeue — is right. Failing the other way would strand live runs as

180 * `running` forever every time a status call timed out.

181 */

182export async function liveRunIds(

183 opts: { url?: string | null; token?: string; fetchImpl?: typeof fetch } = {}

184): Promise<Set<string>> {

185 const url = opts.url ?? runnerUrl();

186 if (!url) return new Set();

187 const doFetch = opts.fetchImpl ?? fetch;

188 const controller = new AbortController();

189 const guard = setTimeout(() => controller.abort(), 5_000);

190 try {

191 const res = await doFetch(`${url}/status`, {

192 headers: { authorization: `Bearer ${opts.token ?? runnerToken()}` },

193 signal: controller.signal,

194 });

195 if (!res.ok) return new Set();

196 const body = (await res.json()) as Partial<RunnerStatus>;

197 return new Set(Array.isArray(body.running) ? body.running.filter((r) => typeof r === "string") : []);

198 } catch {

199 return new Set();

200 } finally {

201 clearTimeout(guard);

202 }

203}

Modifiedsrc/lib/workflow-exec-policy.ts+24−2View fileUnified Split

@@ -32,8 +32,30 @@

3232 * not close it. Anyone on the allowlist still has unsandboxed execution, so

3333 * the list means "people already trusted with production", not "people we

3434 * have decided to let run CI". The actual fix is per-job isolation — see

35 * docs/AUDIT-CI-RUNNER-ISOLATION.md — and this file should be deleted the

36 * day that lands, not extended.

35 * docs/AUDIT-CI-RUNNER-ISOLATION.md.

36 *

37 * ── Status update, 2026-08-31 ────────────────────────────────────────────

38 *

39 * A sibling runner container now exists (src/lib/runner-rpc.ts,

40 * src/runner-agent.ts). When RUNNER_URL and RUNNER_TOKEN are both set, a

41 * step executes there instead: no .env, no /data/repos, its own PID

42 * namespace and its own cgroup. The three paragraphs above become false for

43 * that path — /proc/1/environ holds nothing, and there is no filesystem path

44 * to any repository.

45 *

46 * THIS FILE STAYS ANYWAY, for two reasons, and both must be false before it

47 * goes:

48 *

49 * 1. The runner is off by default and the in-process path is still live.

50 * On an instance that has not enabled it, every word above is current.

51 * 2. Even enabled, one long-lived runner has no boundary BETWEEN jobs. Two

52 * runs share a filesystem and a process namespace, so one tenant's step

53 * can wait for another's checkout. That is a smaller hole than the one

54 * described above, and it is still a hole.

55 *

56 * Delete this file when jobs are isolated from EACH OTHER, not when they are

57 * merely isolated from the app. Relaxing it before then would trade a

58 * documented restriction for an undocumented one.

3759 *

3860 * ── Fail closed ──────────────────────────────────────────────────────────

3961 *

Modifiedsrc/lib/workflow-runner.ts+112−14View fileUnified Split

@@ -16,7 +16,13 @@

1616 * - startWorker({ interval }) — background poll loop (returns stop fn)

1717 */

1818import { and, asc, eq, lt, sql } from "drizzle-orm";

19import { mkdtemp, rm } from "fs/promises";

19import { mkdir, mkdtemp, rm } from "fs/promises";

20import {

21 execViaRunner,

22 liveRunIds,

23 runnerUrl,

24 type ExecOutcome,

25} from "./runner-rpc";

2026import { cpus, loadavg, tmpdir } from "os";

2127import { join } from "path";

2228import { config } from "./config";

@@ -314,15 +320,20 @@ export async function reapStuckRuns(now: Date = new Date()): Promise<number> {

314320 )

315321 )

316322 .limit(50);

317 for (const row of stuck) {

323 // Same guard as the restart sweep, at hour scale: a genuinely long build

324 // on the sibling runner is still a live process, and reaping it would

325 // report a run as abandoned while its output is still arriving.

326 const live = await liveRunIds();

327 const dead = stuck.filter((row) => !live.has(row.id));

328 for (const row of dead) {

318329 await markRunFailed(row.id, "abandoned");

319330 }

320 if (stuck.length > 0) {

331 if (dead.length > 0) {

321332 console.error(

322 `[workflow-runner] reaped ${stuck.length} run(s) abandoned in 'running'`

333 `[workflow-runner] reaped ${dead.length} run(s) abandoned in 'running'`

323334 );

324335 }

325 return stuck.length;

336 return dead.length;

326337 } catch (err) {

327338 console.error("[workflow-runner] reapStuckRuns:", err);

328339 return 0;

@@ -381,8 +392,27 @@ export async function requeueRunsOrphanedByRestart(

381392 )

382393 .limit(50);

383394

395 // With a sibling runner the executing process does NOT die when the app

396 // does: a redeploy recreates the app container while the runner keeps

397 // executing. Without this guard the sweep would mark those live runs

398 // `runner_restarted` and enqueue a second attempt — two executions of one

399 // job against one checkout, racing each other's files. Ask the runner

400 // what it is actually doing.

401 //

402 // Unreachable runner returns an empty set, i.e. today's behaviour: if the

403 // runner is down its runs really are orphaned and requeueing them is

404 // right. The dangerous direction is the other one — stranding live runs

405 // as `running` forever because a status call timed out.

406 const live = await liveRunIds();

407

384408 let requeued = 0;

385409 for (const run of orphans) {

410 if (live.has(run.id)) {

411 console.error(

412 `[workflow-runner] restart sweep: run ${run.id} is still executing on the runner — leaving it alone`

413 );

414 continue;

415 }

386416 // One retry per (workflow, commit): a prior runner_restarted row for

387417 // this pair means this orphan IS the retry — fail it, don't chain.

388418 let alreadyRetried = false;

@@ -529,6 +559,46 @@ async function markRunDone(

529559 * working — when no secrets are loaded the substitution pass is a no-op

530560 * because the regex matches nothing.

531561 */

562/**

563 * Turn a raw execution outcome into the row we persist.

564 *

565 * Shared by both execution paths so a step reported by the sibling runner and

566 * a step spawned in-process are indistinguishable downstream — the isolation

567 * change must not alter what a user sees on a run page.

568 *

569 * A `launchError` is OUR failure, not the user's code, and is labelled in the

570 * output as such. docs/CI-ATTRIBUTION-GUARANTEE.md is a promise that we say so

571 * when a build fails because of us; a runner we could not reach is exactly

572 * that case, and reporting it as a bare non-zero exit would break the promise

573 * in the one place nobody would look.

574 */

575function outcomeToStepResult(

576 name: string,

577 run: string,

578 outcome: ExecOutcome,

579 durationMs: number

580): StepResult {

581 const stderr = outcome.launchError

582 ? truncate(

583 `${outcome.stderr}

584[gluecron] ${outcome.launchError} — this is a platform failure, not your code`,

585 STEP_STREAM_CAP_BYTES

586 )

587 : truncate(outcome.stderr, STEP_STREAM_CAP_BYTES);

588 return {

589 name,

590 run,

591 exitCode: outcome.exitCode,

592 durationMs,

593 stdout: truncate(outcome.stdout, STEP_STREAM_CAP_BYTES),

594 stderr,

595 status:

596 outcome.exitCode === 0 && !outcome.timedOut && !outcome.launchError

597 ? "success"

598 : "failure",

599 };

600}

601

532602async function runStep(

533603 step: ParsedStep,

534604 checkoutDir: string,

@@ -564,6 +634,34 @@ async function runStep(

564634 };

565635 }

566636

637 // SECURITY: do NOT inherit the platform's process.env — a user's `run:`

638 // step would otherwise read every host secret. Only the curated allowlist

639 // plus the runner's own CI vars. See buildRunnerEnv.

640 const stepEnv = buildRunnerEnv({

641 CI: "true",

642 GLUECRON_RUN: runId,

643 GLUECRON_CI: "1",

644 });

645

646 // When a sibling runner container is configured, the step executes THERE:

647 // its own cgroup (so the site's memory budget stops bounding a build), its

648 // own PID namespace (so /proc/1/environ holds nothing worth reading), and

649 // no mount of /data/repos at all. See src/lib/runner-rpc.ts.

650 //

651 // Both paths stay live rather than a flag day: RUNNER_URL unset falls

652 // through to the in-process spawn below, which is what dev machines and

653 // self-hosters without a second container keep using.

654 if (runnerUrl()) {

655 const outcome = await execViaRunner({

656 runId,

657 run,

658 cwd: checkoutDir,

659 env: stepEnv,

660 timeoutMs: STEP_TIMEOUT_MS,

661 });

662 return outcomeToStepResult(name, run, outcome, Date.now() - started);

663 }

664

567665 let proc: ReturnType<typeof Bun.spawn> | null = null;

568666 let timedOut = false;

569667 let killTimer: ReturnType<typeof setTimeout> | null = null;

@@ -591,14 +689,7 @@ async function runStep(

591689 cwd: checkoutDir,

592690 stdout: "pipe",

593691 stderr: "pipe",

594 // SECURITY: do NOT inherit the platform's process.env here — a user's

595 // `run:` step would otherwise read every host secret. Pass only the

596 // curated allowlist + the runner's own CI vars. See buildRunnerEnv.

597 env: buildRunnerEnv({

598 CI: "true",

599 GLUECRON_RUN: runId,

600 GLUECRON_CI: "1",

601 }),

692 env: stepEnv,

602693 });

603694

604695 killTimer = setTimeout(() => {

@@ -677,9 +768,16 @@ async function cloneAt(

677768 commitSha: string | null,

678769 ref: string | null

679770): Promise<{ dir: string } | { error: string }> {

771 // The checkout must land somewhere the RUNNER can see. With a sibling

772 // runner that is the shared `ci-work` volume, mounted at the same path on

773 // both containers; without one it is the ordinary temp dir, unchanged.

774 // This is the only thing the two containers share on the filesystem — the

775 // repository itself is never mounted into the runner, which is the point.

776 const workRoot = process.env["GLUECRON_CI_WORKDIR"] || tmpdir();

680777 let dir: string;

681778 try {

682 dir = await mkdtemp(join(tmpdir(), "gluecron-run-"));

779 if (workRoot !== tmpdir()) await mkdir(workRoot, { recursive: true });

780 dir = await mkdtemp(join(workRoot, "gluecron-run-"));

683781 } catch (err) {

684782 return { error: `mkdtemp failed: ${(err as Error).message}` };

685783 }

Addedsrc/runner-agent.ts+222−0View fileUnified Split

@@ -0,0 +1,222 @@

1/**

2 * The CI runner agent — a second container that executes workflow steps.

3 *

4 * Read src/lib/runner-rpc.ts first; it holds the reasoning. This file is the

5 * other end of that wire, and it is deliberately the most boring program in

6 * the repository:

7 *

8 * - It imports NOTHING from src/db, src/lib/config, or anything that would

9 * pull a database handle or a credential into this process. That absence

10 * is the security property. `/proc/1/environ` in this container is worth

11 * reading and finds a PATH and a token that can only ask this same

12 * process to run a shell command it was already going to run.

13 * - It has no `env_file` in compose, no `/data/repos` mount, and no route

14 * that reads a path it was not given.

15 *

16 * If someone later adds `import { db } from "./db"` here to record a result

17 * directly, every one of those properties is gone in one line. The app records

18 * results; the runner runs commands and answers with what happened.

19 */

21import { spawn } from "node:child_process";

22import { existsSync } from "node:fs";

23import type { ExecRequest, ExecOutcome } from "./lib/runner-rpc";

25const PORT = Number(process.env["RUNNER_PORT"] ?? 3001);

26const TOKEN = process.env["RUNNER_TOKEN"] ?? "";

27const STREAM_CAP_BYTES = 200_000;

28const KILL_GRACE_MS = 5_000;

29/** Refuse a request asking for longer than this, whatever it claims. */

30const MAX_TIMEOUT_MS = 60 * 60_000;

32/** Run ids with a step executing right now. Read by GET /status. */

33const inFlight = new Map<string, number>();

35function truncate(value: string, limit: number): string {

36 return value.length <= limit ? value : value.slice(0, limit) + "\n[... truncated ...]";

37}

39/**

40 * Constant-time-ish bearer check.

41 *

42 * The token guards the ability to run arbitrary shell commands in this

43 * container, so it is the one credential this process does hold. It is

44 * generated per-deploy and never leaves the private docker network.

45 */

46function authorized(req: Request): boolean {

47 if (!TOKEN) return false; // no token configured = refuse everything, loudly

48 const header = req.headers.get("authorization") ?? "";

49 const supplied = header.startsWith("Bearer ") ? header.slice(7) : "";

50 if (supplied.length !== TOKEN.length) return false;

51 let diff = 0;

52 for (let i = 0; i < TOKEN.length; i++) {

53 diff |= supplied.charCodeAt(i) ^ TOKEN.charCodeAt(i);

54 }

55 return diff === 0;

56}

58/**

59 * Execute one step and describe the outcome.

60 *

61 * `nice -n 10` is kept even though this container has its own cgroup: the two

62 * containers still share the host's CPUs, and the app serving traffic should

63 * win a tie against a build. `bash -ec`, so a failing line in a multi-line

64 * script fails the step (the `-e` is load-bearing — without it the step's exit

65 * code is the LAST command's and a build that failed in the middle reports

66 * success).

67 */

68async function execStep(req: ExecRequest): Promise<ExecOutcome> {

69 if (!existsSync(req.cwd)) {

70 // The single most likely misconfiguration, named precisely. Without this

71 // the failure surfaces as a bash ENOENT that sends someone hunting

72 // through their workflow file for a bug that is in our compose file.

73 return {

74 exitCode: null,

75 stdout: "",

76 stderr: "",

77 timedOut: false,

78 launchError:

79 `checkout ${req.cwd} is not visible to the runner container — ` +

80 `the ci-work volume must be mounted at the same path on both services`,

81 };

82 }

84 const timeoutMs = Math.min(Math.max(req.timeoutMs, 1_000), MAX_TIMEOUT_MS);

85 const started = Date.now();

86 inFlight.set(req.runId, started);

88 try {

89 return await new Promise<ExecOutcome>((resolve) => {

90 let stdout = "";

91 let stderr = "";

92 let timedOut = false;

93 let settled = false;

95 const child = spawn("nice", ["-n", "10", "bash", "-ec", req.run], {

96 cwd: req.cwd,

97 env: req.env,

98 stdio: ["ignore", "pipe", "pipe"],

99 });

100

101 const finish = (outcome: ExecOutcome) => {

102 if (settled) return;

103 settled = true;

104 clearTimeout(killTimer);

105 clearTimeout(escalateTimer);

106 resolve(outcome);

107 };

108

109 child.stdout?.on("data", (c: Buffer) => {

110 if (stdout.length < STREAM_CAP_BYTES * 2) stdout += c.toString();

111 });

112 child.stderr?.on("data", (c: Buffer) => {

113 if (stderr.length < STREAM_CAP_BYTES * 2) stderr += c.toString();

114 });

115

116 let escalateTimer: ReturnType<typeof setTimeout> = setTimeout(() => {}, 0);

117 const killTimer = setTimeout(() => {

118 timedOut = true;

119 try {

120 child.kill("SIGTERM");

121 } catch {

122 /* already gone */

123 }

124 escalateTimer = setTimeout(() => {

125 try {

126 child.kill("SIGKILL");

127 } catch {

128 /* already gone */

129 }

130 }, KILL_GRACE_MS);

131 }, timeoutMs);

132

133 child.on("error", (err) => {

134 finish({

135 exitCode: null,

136 stdout: truncate(stdout, STREAM_CAP_BYTES),

137 stderr: truncate(stderr, STREAM_CAP_BYTES),

138 timedOut,

139 launchError: `step failed to launch: ${err.message}`,

140 });

141 });

142

143 child.on("close", (code) => {

144 finish({

145 exitCode: typeof code === "number" ? code : null,

146 stdout: truncate(stdout, STREAM_CAP_BYTES),

147 stderr: truncate(

148 timedOut ? `${stderr}\n[step killed after ${timeoutMs}ms timeout]` : stderr,

149 STREAM_CAP_BYTES

150 ),

151 timedOut,

152 });

153 });

154 });

155 } finally {

156 inFlight.delete(req.runId);

157 }

158}

159

160function json(body: unknown, status = 200): Response {

161 return new Response(JSON.stringify(body), {

162 status,

163 headers: { "content-type": "application/json" },

164 });

165}

166

167const server = Bun.serve({

168 port: PORT,

169 // A step may legitimately run for the full step timeout, and Bun's default

170 // request idle timeout would cut the connection long before that — the app

171 // would then report "runner did not answer" for a step that was running

172 // perfectly well.

173 idleTimeout: 255,

174 async fetch(req) {

175 const url = new URL(req.url);

176

177 // Unauthenticated: liveness only, so docker's healthcheck needs no secret.

178 if (url.pathname === "/healthz") return new Response("ok");

179

180 if (!authorized(req)) return json({ error: "unauthorized" }, 401);

181

182 if (url.pathname === "/status" && req.method === "GET") {

183 return json({ running: [...inFlight.keys()] });

184 }

185

186 if (url.pathname === "/exec" && req.method === "POST") {

187 let body: ExecRequest;

188 try {

189 body = (await req.json()) as ExecRequest;

190 } catch {

191 return json({ error: "invalid json" }, 400);

192 }

193 if (

194 typeof body?.runId !== "string" ||

195 typeof body?.run !== "string" ||

196 typeof body?.cwd !== "string"

197 ) {

198 return json({ error: "runId, run and cwd are required" }, 400);

199 }

200 const outcome = await execStep({

201 runId: body.runId,

202 run: body.run,

203 cwd: body.cwd,

204 env: body.env && typeof body.env === "object" ? body.env : {},

205 timeoutMs: typeof body.timeoutMs === "number" ? body.timeoutMs : 15 * 60_000,

206 });

207 return json(outcome);

208 }

209

210 return json({ error: "not found" }, 404);

211 },

212});

213

214if (!TOKEN) {

215 // Refusing every request while looking healthy is the worst of both worlds,

216 // so say it at the only moment anyone is reading: startup.

217 console.error(

218 "[runner-agent] RUNNER_TOKEN is not set — every /exec will be refused. " +

219 "Set it identically on the app and the runner."

220 );

221}

222console.log(`[runner-agent] listening on :${server.port} (no database, no secrets)`);

0223

c comment · e edit title · m merge · a approve · r request changes · ? shortcuts