Code Issues Pull Requests Actions Security InsightsSettings

feat(deploy): failed deploys and rollbacks notify the owner #5472

Merged⚡ AI-generatedXSccantynz wants to mergefeat/deploy-failure-alerts→mainopened 23d ago

4 changed files+120−1

Modifiedscripts/auto-update.sh+18−0View fileUnified Split

@@ -27,6 +27,17 @@ COMPOSE="docker compose -f docker-compose.standalone.yml"

2727IMAGE="gluecron-gluecron"

2828FAILED_MARKER="$REPO_DIR/.last-failed-deploy-sha"

2929

30# Owner notification for failed deploys, via the app container (it has the

31# DB + email credentials this host script does not). Best-effort by design:

32# on the rollback paths the container runs the ROLLED-BACK image, which may

33# predate scripts/notify-deploy-event.ts — and on the catastrophic paths the

34# container may be dead entirely. `|| true` covers both; the external

35# heartbeat workflow (GitHub mirror) and FAILED_MARKER remain the backstop.

36notify_owner() {

37 docker exec gluecron-gluecron-1 bun scripts/notify-deploy-event.ts "$@" 2>/dev/null \

38 || echo "$(date -Is) owner notification for '$1' could not be delivered (container image predates it, or container down)" >&2

39}

3041cd "$REPO_DIR"

3142git fetch origin "$BRANCH" --quiet

3243

@@ -146,6 +157,7 @@ docker logs --tail 80 gluecron-gluecron-1 >&2 || true

146157if ! docker image inspect "$IMAGE:last-good" >/dev/null 2>&1; then

147158 echo "$(date -Is) no last-good image to roll back to (first deploy on this box?) — human intervention required" >&2

148159 echo "$remote_sha" > "$FAILED_MARKER"

160 notify_owner no_last_good "$remote_sha"

149161 exit 1

150162fi

151163

@@ -174,8 +186,14 @@ echo "$remote_sha" > "$FAILED_MARKER"

174186

175187if [ "$rb_healthy" = "1" ]; then

176188 echo "$(date -Is) rollback to $prev_sha succeeded — app healthy again. $remote_sha will not be retried automatically; push a fix or clear $FAILED_MARKER to try again." >&2

189 if [ "$migrate_ok" = "0" ]; then

190 notify_owner rollback_ok "$remote_sha" "migrations did not apply"

191 else

192 notify_owner rollback_ok "$remote_sha" "app health/readiness never came up"

193 fi

177194 exit 1

178195fi

179196

180197echo "$(date -Is) ROLLBACK ALSO FAILED — human intervention required NOW (site may be down)" >&2

198notify_owner rollback_failed "$remote_sha"

181199exit 1

Addedscripts/notify-deploy-event.ts+64−0View fileUnified Split

@@ -0,0 +1,64 @@

1#!/usr/bin/env bun

2/**

3 * Deploy-event owner notification — called by scripts/auto-update.sh via

4 * `docker exec` so it runs INSIDE the app container, which has the DB and

5 * email credentials the host script does not.

6 *

7 * Why this exists: the automatic rollback (2026-07-21) worked and was

8 * silent. A failed deploy rolled back into journalctl, the FAILED_MARKER

9 * quietly blocked the sha from retrying, and the owner's next experience

10 * was "I keep pushing and nothing ships" with no visible reason. Rollback

11 * without notification is half a rollback.

12 *

13 * Caveat, deliberate: on the rollback paths this executes in the ROLLED-

14 * BACK image, which may predate this script (first rollout, or an old

15 * last-good). auto-update.sh calls it with `|| true`; the external

16 * heartbeat workflow and the failure marker remain the backstop.

17 *

18 * Usage: bun scripts/notify-deploy-event.ts <event> <sha> [detail...]

19 * event: rollback_ok | rollback_failed | no_last_good

20 */

21import { fanOutSpineAlert } from "../src/lib/spine-alert-fanout";

23const [event = "", sha = "", ...rest] = process.argv.slice(2);

24const short = sha.slice(0, 7) || "unknown";

25const detail = rest.join(" ");

27const messages: Record<string, { title: string; body: string }> = {

28 rollback_ok: {

29 title: `🟠 Deploy of ${short} FAILED — rolled back automatically`,

30 body:

31 `The deploy of ${sha} failed (${detail || "see journalctl -u gluecron-update"}) and the box ` +

32 `rolled back to the previous release. The site is up on the OLD code.\n\n` +

33 `This commit will NOT retry automatically — push a fix, or clear ` +

34 `/opt/gluecron/.last-failed-deploy-sha to retry the same sha.`,

35 },

36 rollback_failed: {

37 title: `🔴 Deploy of ${short} failed AND rollback failed — site may be down`,

38 body:

39 `Both the deploy of ${sha} and the rollback to the previous release failed ` +

40 `their health gates. Human intervention required now: ` +

41 `ssh root@100.109.131.122, then journalctl -u gluecron-update and ` +

42 `docker compose -f docker-compose.standalone.yml logs gluecron.`,

43 },

44 no_last_good: {

45 title: `🔴 Deploy of ${short} failed with NO rollback image`,

46 body:

47 `The deploy of ${sha} failed its health gate and there is no last-good ` +

48 `image to roll back to (first deploy on this box?). Human intervention required.`,

49 },

50};

52const msg = messages[event];

53if (!msg) {

54 console.error(`[notify-deploy-event] unknown event "${event}"`);

55 process.exit(1);

56}

58await fanOutSpineAlert({

59 kind: "deploy_failed",

60 title: msg.title,

61 body: msg.body,

62 url: "/admin/deploys",

63});

64process.exit(0);

Modifiedsrc/__tests__/auto-update-deploy.test.ts+37−0View fileUnified Split

@@ -186,6 +186,43 @@ describe("the readiness gate", () => {

186186 });

187187});

188188

189describe("failed deploys notify the owner", () => {

190 // The rollback (2026-07-21) worked and was SILENT: a failed deploy rolled

191 // back into journalctl, FAILED_MARKER quietly blocked the sha, and the

192 // owner's next experience was "I keep pushing and nothing ships" with no

193 // visible reason. Rollback without notification is half a rollback.

194 test("a rolled-back deploy calls notify-deploy-event with the cause", async () => {

195 const r = await run({ migrate: 1 });

196 expect(r.calls).toContain("notify-deploy-event.ts rollback_ok");

197 expect(r.calls).toContain("migrations did not apply");

198 });

199

200 test("an unhealthy-app rollback reports health as the cause", async () => {

201 const r = await run({ health: 1 });

202 expect(r.calls).toContain("notify-deploy-event.ts rollback_ok");

203 expect(r.calls).toContain("never came up");

204 });

205

206 test("no last-good image is its own, louder event", async () => {

207 const r = await run({ health: 1, lastGood: false });

208 expect(r.calls).toContain("notify-deploy-event.ts no_last_good");

209 });

210

211 test("a healthy deploy sends nothing", async () => {

212 const r = await run({});

213 expect(r.calls).not.toContain("notify-deploy-event");

214 });

215

216 test("notification failure cannot change the script's outcome", async () => {

217 // notify_owner must be || -guarded: on rollback the container runs the

218 // rolled-back image (which may predate the notify script), and on the

219 // catastrophic path it may be dead entirely.

220 const src = await Bun.file("scripts/auto-update.sh").text();

221 const fn = src.slice(src.indexOf("notify_owner()"));

222 expect(fn).toContain("||");

223 });

224});

225

189226describe("no script anywhere swallows a migration failure", () => {

190227 /**

191228 * The sweep, because this was three sites and not one.

Modifiedsrc/lib/spine-alert-fanout.ts+1−1View fileUnified Split

@@ -27,7 +27,7 @@ import { createNotification } from "./notify";

2727import { sendEmail } from "./email";

2828

2929export interface SpineAlert {

30 kind: "check_red" | "check_recovered" | "error_spike";

30 kind: "check_red" | "check_recovered" | "error_spike" | "deploy_failed";

3131 title: string;

3232 body: string;

3333 /** Where the notification links. Defaults to /admin/spine. */

3434

c comment · e edit title · m merge · a approve · r request changes · ? shortcuts