Code Issues Pull Requests Actions Security InsightsSettings

fix(deploy): INCIDENT 2026-08-19 — script died silently after reset without DEPLOY_EVENT_TOKEN; in-container /api probes need X-Forwarded-For; EXIT trap restores HEAD #5505

Merged⚡ AI-generatedXSccantynz wants to mergefix/deploy-script-2026-08-19→mainopened 13d ago

2 changed files+93−3

Modifiedscripts/auto-update.sh+42−2View fileUnified Split

@@ -49,7 +49,13 @@ DEPLOY_T0=$(date +%s)

4949deploy_event() {

5050 local kind="$1" status="${2:-}" err="${3:-}"

5151 local token=""

52 [ -f "$REPO_DIR/.env" ] && token=$(grep -E '^DEPLOY_EVENT_TOKEN=' "$REPO_DIR/.env" | head -1 | cut -d= -f2- | tr -d '"'"'"' \r')

52 # `|| true` is load-bearing: with `set -euo pipefail`, a .env WITHOUT the

53 # token made grep exit 1, which killed the whole script right after the

54 # git reset (INCIDENT 2026-08-19: HEAD advanced, nothing deployed, and the

55 # timer then saw "nothing new" forever). Missing token = skip, never die.

56 if [ -f "$REPO_DIR/.env" ]; then

57 token=$( (grep -E '^DEPLOY_EVENT_TOKEN=' "$REPO_DIR/.env" || true) | head -1 | cut -d= -f2- | tr -d '"'"'"' \r' || true)

58 fi

5359 [ -z "$token" ] && return 0

5460 local body

5561 if [ "$kind" = "started" ]; then

@@ -58,7 +64,11 @@ deploy_event() {

5864 local dur=$(( ( $(date +%s) - DEPLOY_T0 ) * 1000 ))

5965 body="{\"run_id\":\"$DEPLOY_RUN_ID\",\"status\":\"$status\",\"duration_ms\":$dur,\"error\":\"$err\"}"

6066 fi

67 # X-Forwarded-For: inside the container there is no proxy in front of the

68 # app, and the /api/* rate limiter refuses requests whose client address it

69 # cannot determine (400). Loopback is the honest answer here.

6170 docker exec gluecron-gluecron-1 wget -qO- --timeout=5 \

71 --header="X-Forwarded-For: 127.0.0.1" \

6272 --header="authorization: Bearer $token" \

6373 --header="content-type: application/json" \

6474 --post-data="$body" \

@@ -66,6 +76,27 @@ deploy_event() {

6676}

6777

6878cd "$REPO_DIR"

80# Safety net for the unexpected-exit class (a bug in THIS script, a killed

81# process): once `git reset --hard` has advanced HEAD, any exit before

82# "deploy complete" would otherwise leave the repo at the new sha with the

83# OLD container running — and the next tick sees local == remote and never

84# retries. The trap puts HEAD back so the next tick tries again, and says so.

85DEPLOY_DONE=0

86prev_sha=""

87on_exit() {

88 local rc=$?

89 if [ "$DEPLOY_DONE" = "0" ] && [ -n "$prev_sha" ]; then

90 local head_now

91 head_now=$(git rev-parse HEAD 2>/dev/null || echo "")

92 if [ -n "$head_now" ] && [ "$head_now" != "$prev_sha" ]; then

93 echo "$(date -Is) deploy did not complete (exit $rc) — restoring HEAD to $prev_sha so the next tick retries" >&2

94 git reset --hard "$prev_sha" >/dev/null 2>&1 || true

95 fi

96 fi

97}

98trap on_exit EXIT

69100git fetch origin "$BRANCH" --quiet

70101

71102local_sha=$(git rev-parse HEAD)

@@ -119,6 +150,7 @@ if [ "$build_ok" = "0" ]; then

119150 git reset --hard "$prev_sha"

120151 echo "$remote_sha" > "$FAILED_MARKER"

121152 deploy_event finished failed "image build failed"

153 DEPLOY_DONE=1

122154 notify_owner build_failed "$remote_sha"

123155 exit 1

124156fi

@@ -130,6 +162,7 @@ if [ "$migrate_ok" = "0" ]; then

130162 git reset --hard "$prev_sha"

131163 echo "$remote_sha" > "$FAILED_MARKER"

132164 deploy_event finished failed "migrations did not apply"

165 DEPLOY_DONE=1

133166 notify_owner migration_aborted "$remote_sha" "migrations did not apply"

134167 exit 1

135168fi

@@ -182,7 +215,10 @@ done

182215sha_mismatch=0

183216served_sha=""

184217if [ "$healthy" = "1" ]; then

185 served_json=$(docker exec gluecron-gluecron-1 wget -qO- --timeout=4 http://localhost:3000/api/version 2>/dev/null || true)

218 # Same X-Forwarded-For reason as deploy_event: without it /api/version

219 # answers 400 inside the container and the gate degraded to "could not

220 # read served sha" on every deploy (2026-08-19 journal).

221 served_json=$(docker exec gluecron-gluecron-1 wget -qO- --timeout=4 --header="X-Forwarded-For: 127.0.0.1" http://localhost:3000/api/version 2>/dev/null || true)

186222 served_sha=$(printf '%s' "$served_json" | sed -n 's/.*"shaFull":"$[0-9a-f]*$".*/\1/p')

187223 [ -z "$served_sha" ] && served_sha=$(printf '%s' "$served_json" | sed -n 's/.*"sha":"$[0-9a-f]*$".*/\1/p')

188224 case "$served_sha" in

@@ -204,6 +240,7 @@ if [ "$healthy" = "1" ] && [ "$migrate_ok" = "1" ] && [ "$sha_mismatch" = "0" ];

204240 # the failure that corrupts pushes and kills backups; `image prune -f`

205241 # above only removes dangling images, never the BuildKit cache.

206242 docker builder prune -af --filter until=72h >/dev/null 2>&1 || true

243 DEPLOY_DONE=1

207244 deploy_event finished succeeded

208245 echo "$(date -Is) deploy complete: $remote_sha (app healthy, migrations applied, serving $served_sha)"

209246 exit 0

@@ -220,6 +257,7 @@ if ! docker image inspect "$IMAGE:last-good" >/dev/null 2>&1; then

220257 echo "$(date -Is) no last-good image to roll back to (first deploy on this box?) — human intervention required" >&2

221258 echo "$remote_sha" > "$FAILED_MARKER"

222259 deploy_event finished failed "health gate failed; no last-good image"

260 DEPLOY_DONE=1

223261 notify_owner no_last_good "$remote_sha"

224262 exit 1

225263fi

@@ -247,6 +285,7 @@ done

247285

248286echo "$remote_sha" > "$FAILED_MARKER"

249287

288DEPLOY_DONE=1

250289if [ "$rb_healthy" = "1" ]; then

251290 deploy_event finished failed "health gate failed; rolled back to $prev_sha"

252291 echo "$(date -Is) rollback to $prev_sha succeeded — app healthy again. $remote_sha will not be retried automatically; push a fix or clear $FAILED_MARKER to try again." >&2

@@ -258,6 +297,7 @@ if [ "$rb_healthy" = "1" ]; then

258297 exit 1

259298fi

260299

300DEPLOY_DONE=1

261301deploy_event finished failed "health gate failed AND rollback failed"

262302echo "$(date -Is) ROLLBACK ALSO FAILED — human intervention required NOW (site may be down)" >&2

263303notify_owner rollback_failed "$remote_sha"

Modifiedsrc/__tests__/auto-update-deploy.test.ts+51−1View fileUnified Split

@@ -35,6 +35,11 @@ afterAll(() => {

3535});

3636

3737interface Stubs {

38 /** Make `git rev-parse HEAD` fail once HEAD has been advanced — an

39 * unexpected death after the reset, to exercise the EXIT trap. */

40 crashAfterReset?: boolean;

41 /** Write a .env into REPO_DIR (contents given). Simulates the box. */

42 envFile?: string;

3843 /** Exit code for `db:migrate`. */

3944 migrate?: number;

4045 /** Exit code for the /healthz and /readyz probes (forward gate only —

@@ -58,6 +63,7 @@ function setup(stubs: Stubs): { repo: string; bin: string } {

5863 const bin = join(dir, "bin");

5964 mkdirSync(repo, { recursive: true });

6065 mkdirSync(bin, { recursive: true });

66 if (stubs.envFile !== undefined) writeFileSync(join(repo, ".env"), stubs.envFile, "utf8");

6167

6268 const sh = (name: string, body: string) => {

6369 const p = join(bin, name);

@@ -67,10 +73,20 @@ function setup(stubs: Stubs): { repo: string; bin: string } {

6773

6874 // Local HEAD and origin/main differ, so the script proceeds past its

6975 // "nothing new" early exit.

76 // Stateful: `reset --hard X` records X as HEAD so a later `rev-parse HEAD`

77 // answers it — that is what lets the EXIT-trap test observe HEAD being

78 // advanced by the deploy and restored on an unexpected death.

79 const state = join(repo, "head.state").replace(/\\/g, "/");

7080 sh(

7181 "git",

7282 `case "$*" in

73 *"rev-parse HEAD"*) echo "aaaaaaa";;

83 *"reset --hard origin/main"*) echo "bbbbbbb" > "${state}";;

84 *"reset --hard "*) echo "$*" | sed 's/.*reset --hard //' > "${state}";;

85 *"rev-parse HEAD"*)

86 if [ -n "\${CRASH_AFTER_RESET:-}" ] && [ -f "${state}" ] && [ "$(cat "${state}")" = "bbbbbbb" ] && [ ! -f "${state}.crashed" ]; then

87 touch "${state}.crashed"; echo "git $*" >> "${repo.replace(/\\/g, "/")}/calls.log"; exit 1

88 fi

89 if [ -f "${state}" ]; then cat "${state}"; else echo "aaaaaaa"; fi;;

7490 *"rev-parse origin/main"*) echo "bbbbbbb";;

7591 *) : ;;

7692 esac

@@ -107,6 +123,7 @@ async function run(stubs: Stubs) {

107123 REPO_DIR: repo,

108124 HEALTH_TRIES: "2",

109125 HEALTH_SLEEP: "0",

126 ...(stubs.crashAfterReset ? { CRASH_AFTER_RESET: "1" } : {}),

110127 },

111128 stdout: "pipe",

112129 stderr: "pipe",

@@ -244,6 +261,39 @@ describe("failed deploys notify the owner", () => {

244261 expect(r.marker).toBe(true);

245262 });

246263

264 test("a .env WITHOUT DEPLOY_EVENT_TOKEN must not kill the deploy (INCIDENT 2026-08-19)", async () => {

265 // grep exit 1 + set -euo pipefail killed the script right after git reset:

266 // HEAD advanced, nothing deployed, and the timer saw 'nothing new' forever.

267 const r = await run({ envFile: "DATABASE_URL=postgres://x\nANTHROPIC_API_KEY=\n" });

268 expect(r.exitCode).toBe(0);

269 expect(r.out).toContain("deploy complete");

270 expect(r.calls).not.toContain("api/events/deploy");

271 });

272

273 test("with a token the started/finished deploy events are posted with X-Forwarded-For", async () => {

274 const r = await run({ envFile: "DEPLOY_EVENT_TOKEN=\"abc123\"\n" });

275 expect(r.exitCode).toBe(0);

276 expect(r.calls).toContain("api/events/deploy/started");

277 expect(r.calls).toContain("api/events/deploy/finished");

278 expect(r.calls).toContain("X-Forwarded-For: 127.0.0.1");

279 expect(r.calls).toContain("authorization: Bearer abc123");

280 });

281

282 test("served-sha probe sends X-Forwarded-For so the in-container rate limiter answers", async () => {

283 const r = await run({});

284 expect(r.calls).toMatch(/X-Forwarded-For: 127\.0\.0\.1 http:\/\/localhost:3000\/api\/version/);

285 });

286

287 test("an unexpected death after the reset restores HEAD so the next tick retries (INCIDENT 2026-08-19)", async () => {

288 const r = await run({ crashAfterReset: true });

289 expect(r.exitCode).not.toBe(0);

290 expect(r.out).toContain("deploy did not complete");

291 expect(r.out).toContain("restoring HEAD to aaaaaaa");

292 // the last reset puts the old sha back

293 const resets = r.calls.split("\n").filter((l) => l.includes("reset --hard"));

294 expect(resets[resets.length - 1]).toContain("reset --hard aaaaaaa");

295 });

296

247297 test("a matching served sha completes and prunes the build cache", async () => {

248298 const r = await run({});

249299 expect(r.exitCode).toBe(0);

250300

c comment · e edit title · m merge · a approve · r request changes · ? shortcuts