chore(hygiene): NUL byte in repository.ts, hourly mirror warning, dead deploy workflows removed #5511
4 changed files+15−646
Deleted.github/workflows/fly-deploy.yml+0−97View fileUnifiedSplit
@@ -1,97 +0,0 @@
1name: Fly Deploy
2
3# Default-deny: scope GITHUB_TOKEN to read-only. Individual jobs can
4# escalate via their own `permissions:` block if they need write access.
5permissions: read-all
6
7on:
8 # SUPERSEDED 2026-05-31 — gluecron.com migrated off Fly.io to a dedicated
9 # Vultr box that self-deploys by pulling from GitHub (see
10 # scripts/standalone-deploy.sh). Manual dispatch only; no longer auto-fires.
11 workflow_dispatch: {}
12
13concurrency:
14 group: fly-deploy-${{ github.ref }}
15 cancel-in-progress: false
16
17jobs:
18 deploy:
19 name: Deploy to Fly.io
20 runs-on: ubuntu-latest
21 steps:
22 - uses: actions/checkout@v4
23
24 - name: Install flyctl
25 run: |
26 curl -fsSL https://github.com/superfly/flyctl/releases/download/v0.4.38/flyctl_0.4.38_Linux_x86_64.tar.gz -o /tmp/flyctl.tgz
27 mkdir -p $HOME/.fly/bin
28 tar -xzf /tmp/flyctl.tgz -C $HOME/.fly/bin
29 echo "$HOME/.fly/bin" >> $GITHUB_PATH
30 $HOME/.fly/bin/flyctl version
31
32 - name: Ensure app exists
33 run: |
34 if ! flyctl apps list --json 2>/dev/null | grep -q '"Name":\s*"gluecron"'; then
35 echo "Creating Fly app 'gluecron'..."
36 flyctl apps create gluecron --org personal
37 else
38 echo "App 'gluecron' already exists."
39 fi
40 env:
41 FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
42
43 - name: Ensure volume exists
44 continue-on-error: true
45 run: |
46 if ! flyctl volumes list -a gluecron --json 2>/dev/null | grep -qi '"name":\s*"gluecron_repos"'; then
47 echo "Creating volume 'gluecron_repos' in iad..."
48 flyctl volumes create gluecron_repos --app gluecron --size 3 --region iad --yes || \
49 echo "Warning: volume creation failed (may already exist or account limit reached). Continuing."
50 else
51 echo "Volume 'gluecron_repos' already exists."
52 fi
53 env:
54 FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
55
56 - name: Sync application secrets to Fly
57 run: |
58 flyctl secrets set --app gluecron --stage \
59 DATABASE_URL="$DATABASE_URL" \
60 APP_BASE_URL="$APP_BASE_URL" \
61 DEMO_SEED_ON_BOOT="$DEMO_SEED_ON_BOOT" \
62 EMAIL_PROVIDER="$EMAIL_PROVIDER" \
63 SITE_ADMIN_USERNAME="$SITE_ADMIN_USERNAME"
64 env:
65 FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
66 DATABASE_URL: ${{ secrets.DATABASE_URL }}
67 APP_BASE_URL: ${{ vars.APP_BASE_URL || 'https://gluecron.fly.dev' }}
68 # Defaults to OFF. It defaulted to '1', which meant a production
69 # deploy created a `demo` user and three fabricated repos, and those
70 # rows were then counted in every /admin stat tile, in the signup
71 # chart and activation rate on /admin/growth, in /admin/users,
72 # /admin/repos and /admin/billing. Both DEPLOY_CHECKLIST.md and
73 # LAUNCH_TODAY.md already described the default as "off"; the code
74 # was the thing that was wrong.
75 DEMO_SEED_ON_BOOT: ${{ vars.DEMO_SEED_ON_BOOT || '0' }}
76 EMAIL_PROVIDER: ${{ vars.EMAIL_PROVIDER || 'log' }}
77 SITE_ADMIN_USERNAME: ${{ vars.SITE_ADMIN_USERNAME || 'ccantynz-alt' }}
78
79 - name: Deploy
80 run: flyctl deploy --remote-only --app gluecron
81 env:
82 FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
83
84 - name: Smoke test
85 run: |
86 sleep 10
87 for i in 1 2 3 4 5; do
88 code=$(curl -s -o /dev/null -w "%{http_code}" https://gluecron.fly.dev/healthz || echo "000")
89 echo "Attempt $i: /healthz returned $code"
90 if [ "$code" = "200" ]; then
91 echo "Site is live."
92 exit 0
93 fi
94 sleep 10
95 done
96 echo "Site did not respond with 200 after 50s. Check 'fly logs -a gluecron'."
97 exit 1
Deleted.github/workflows/hetzner-deploy.yml+0−544View fileUnifiedSplit
@@ -1,544 +0,0 @@
1name: Hetzner Deploy (gluecron.com)
2
3# Default-deny: scope GITHUB_TOKEN to read-only. The deploy uses an SSH key
4# (HETZNER_SSH_KEY) to push to the box, not the GitHub token, so read-only
5# is sufficient. Individual jobs may escalate if they need write access.
6permissions: read-all
7
8# Triggered on every push to main and on manual dispatch.
9# Steps:
10# 1. Capture the current SHA on the box (for rollback)
11# 2. SSH in, git pull, run deploy-crontech.sh (which restarts systemd + reloads Caddy)
12# 3. Smoke-test https://gluecron.com/healthz with retries
13# 4. On smoke failure: roll back to the previous SHA and restart
14# 5. On any failure: have Claude read the last 100 journal lines and post a one-paragraph
15# root-cause analysis to the workflow summary (and optionally a webhook)
16#
17# Secrets required:
18# HETZNER_HOST — public IP/hostname of the box (e.g. 178.104.208.252)
19# HETZNER_USER — ssh user (e.g. root)
20# HETZNER_SSH_KEY — private deploy key (PEM/OpenSSH format)
21#
22# Optional:
23# ANTHROPIC_API_KEY — enables AI failure-diagnosis step
24# DEPLOY_WEBHOOK_URL — POSTed with JSON deploy status (Slack/Discord/anything)
25# DEPLOY_EVENT_TOKEN — bearer used to POST deploy timeline events to
26# ${APP_BASE_URL}/api/events/deploy/{started,finished}.
27# Block N3: this makes the live site display ITS OWN
28# deploy in the admin nav + at /admin/deploys.
29# APP_BASE_URL — base URL of the running site (default https://gluecron.com).
30
31on:
32 # SUPERSEDED 2026-05-31 — gluecron.com now runs on a dedicated Vultr box that
33 # deploys itself by PULLING from GitHub (systemd git-poll timer installed by
34 # scripts/standalone-deploy.sh -> scripts/auto-update.sh). This SSH-push
35 # pipeline targeted the old systemd/localhost:3010 host and was failing with
36 # "dial tcp :22 i/o timeout". Manual dispatch only; no longer auto-fires.
37 workflow_dispatch: {}
38
39concurrency:
40 group: hetzner-deploy
41 cancel-in-progress: false
42
43jobs:
44 deploy:
45 name: Deploy gluecron.com
46 runs-on: ubuntu-latest
47 timeout-minutes: 12
48 steps:
49 - uses: actions/checkout@v4
50
51 # ─── 0. Mark start time + notify the live site (Block N3) ────────────
52 # The site has its own admin status pill + /admin/deploys timeline.
53 # POSTing here makes the pill say "Deploying… 14s" the moment SSH
54 # begins. `--fail` is intentionally NOT set: a 5xx here must never
55 # block the deploy itself.
56 - name: Record start time
57 id: start
58 run: |
59 echo "epoch=$(date +%s)" >> $GITHUB_OUTPUT
60
61 - name: Notify deploy started
62 if: env.DEPLOY_EVENT_TOKEN != ''
63 env:
64 DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
65 APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
66 run: |
67 curl --silent --show-error --max-time 10 \
68 -X POST "$APP_BASE_URL/api/events/deploy/started" \
69 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
70 -H "content-type: application/json" \
71 --data "{\"sha\":\"${{ github.sha }}\",\"run_id\":\"${{ github.run_id }}\",\"source\":\"hetzner-deploy\"}" \
72 || echo "(deploy-started notify failed — continuing)"
73
74 # ─── 1. Capture pre-deploy SHA so we can rollback ───────────────────
75 # R2 — bracket every major step with notify-deploy-step calls so the
76 # /admin/deploys modal can stream the workflow live. The composite
77 # action is fire-and-forget — a 5xx never blocks the deploy.
78 - name: Notify step — setup (in_progress)
79 if: env.DEPLOY_EVENT_TOKEN != ''
80 uses: ./.github/actions/notify-deploy-step
81 with:
82 step_name: setup
83 status: in_progress
84 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
85 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
86
87 - name: Capture pre-deploy SHA
88 id: prev
89 uses: appleboy/ssh-action@v1.2.0
90 with:
91 host: ${{ secrets.HETZNER_HOST }}
92 username: ${{ secrets.HETZNER_USER }}
93 key: ${{ secrets.HETZNER_SSH_KEY }}
94 script_stop: true
95 script: |
96 cd /opt/gluecron
97 sha=$(git rev-parse HEAD)
98 echo "Previous SHA: $sha"
99 echo "$sha" > /tmp/gluecron_prev_sha
100 cat /tmp/gluecron_prev_sha
101
102 - name: Notify step — setup (succeeded)
103 if: success() && env.DEPLOY_EVENT_TOKEN != ''
104 uses: ./.github/actions/notify-deploy-step
105 with:
106 step_name: setup
107 status: succeeded
108 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
109 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
110
111 - name: Notify step — setup (failed)
112 if: failure() && env.DEPLOY_EVENT_TOKEN != ''
113 uses: ./.github/actions/notify-deploy-step
114 with:
115 step_name: setup
116 status: failed
117 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
118 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
119
120 # ─── 2. Deploy: pull main, install/compile/restart ─────────────────
121 # Block N2 — speed optimisation:
122 # (a) Cache `bun install` by hashing bun.lock; skip the walk when
123 # the lockfile is unchanged. Saves ~5-15s per "no deps changed"
124 # deploy.
125 # (b) Compile to a single Bun static binary at
126 # /opt/gluecron/.next/gluecron-server. Boot drops from ~500ms
127 # (cold ESM resolve) to <50ms. Compile itself takes ~3-5s.
128 # (c) Rewrite the systemd unit on first run if it lacks Type=notify
129 # (idempotent — diff-then-write, daemon-reload only on change).
130 # Falls back to `bun run src/index.ts` if the compiled binary is
131 # missing for any reason — the deploy MUST stay backward-
132 # compatible.
133 # (d) `systemctl restart` blocks until sd_notify(READY=1) fires
134 # (wired in src/lib/systemd-notify.ts), so we no longer rely on
135 # the curl-with-retries loop downstream.
136 - name: Deploy
137 id: deploy
138 uses: appleboy/ssh-action@v1.2.0
139 env:
140 DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
141 APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
142 GH_RUN_ID: ${{ github.run_id }}
143 GH_SHA: ${{ github.sha }}
144 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
145 GH_REPO: ${{ github.repository }}
146 with:
147 host: ${{ secrets.HETZNER_HOST }}
148 username: ${{ secrets.HETZNER_USER }}
149 key: ${{ secrets.HETZNER_SSH_KEY }}
150 command_timeout: 8m
151 script_stop: true
152 envs: GH_TOKEN,GH_REPO,GH_SHA
153 # UNIFIED DEPLOY (2026-05-16 reliability sweep, Phase D).
154 #
155 # Both deploy paths now share ONE source of truth:
156 # `scripts/self-deploy.sh`. The GitHub Actions Hetzner
157 # workflow (this file) and the post-receive hook on the
158 # gluecron-hosted git server both invoke the same script.
159 # Drift between deploy paths — flagged in AUDIT-v2.md P1
160 # #11 — is eliminated.
161 #
162 # The script handles: bun install, db migrations, bun build
163 # --compile, BUILD_SHA pin into systemd drop-in, systemctl
164 # restart, healthz wait, post-deploy smoke suite, AND
165 # automatic rollback to the previous SHA on failure.
166 #
167 # All we do here:
168 # 1. Repair the git remote (still needed because the box
169 # historically pointed at the self-hosted URL which 404s).
170 # 2. git fetch + reset --hard to the new SHA.
171 # 3. `bash scripts/self-deploy.sh --inline` (the `--inline`
172 # flag prevents the script's normal background re-exec
173 # so the SSH session blocks until completion, which is
174 # what we want for GH Actions to mirror the script's
175 # exit code).
176 script: |
177 set -Eeuxo pipefail
178 cd /opt/gluecron
179
180 echo "=== STEP 1: repair git remote + fetch + reset ==="
181 git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${GH_REPO}.git"
182 git fetch --prune origin main
183 git reset --hard origin/main
184 new_sha=$(git rev-parse HEAD)
185 echo "Deployed SHA on box: $new_sha"
186
187 echo "=== STEP 2: hand off to scripts/self-deploy.sh (the canonical deploy path) ==="
188 chmod +x scripts/self-deploy.sh
189 bash scripts/self-deploy.sh --inline
190 echo "=== DONE: self-deploy.sh exited cleanly ==="
191
192 # ─── 3. Smoke-test the deployed app on the box ──────────────────────
193 # We SSH back in and curl localhost:3010/healthz directly. This tests
194 # the EXACT instance we just deployed, independent of:
195 # - DNS state (gluecron.com may still point at an old box during a
196 # migration)
197 # - Caddy TLS state (cert may not be issued yet for a new domain)
198 # - external network reachability from GH runners
199 # If you ALSO want a public-DNS smoke check, add a second step that
200 # hits https://gluecron.com after this one succeeds.
201 - name: Notify step — smoke-test (in_progress)
202 if: env.DEPLOY_EVENT_TOKEN != ''
203 uses: ./.github/actions/notify-deploy-step
204 with:
205 step_name: smoke-test
206 status: in_progress
207 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
208 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
209
210 - name: Smoke test (localhost on the box)
211 id: smoke
212 uses: appleboy/ssh-action@v1.2.0
213 with:
214 host: ${{ secrets.HETZNER_HOST }}
215 username: ${{ secrets.HETZNER_USER }}
216 key: ${{ secrets.HETZNER_SSH_KEY }}
217 script_stop: true
218 script: |
219 # Block N2 — `systemctl restart` already blocked on
220 # sd_notify(READY=1), so the FIRST curl should succeed. We keep a
221 # short retry budget for paranoia: a brief delay between
222 # systemd's READY ack and the HTTP listener becoming routable
223 # via 127.0.0.1 is theoretically possible (unusual but cheap to
224 # tolerate). 3 attempts × 2s = 6s ceiling instead of 8 × 6s = 48s.
225 set +e
226 for i in 1 2 3; do
227 code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
228 echo "Attempt $i: /healthz -> $code"
229 if [ "$code" = "200" ]; then
230 echo "OK: gluecron is healthy on localhost:3010"
231 curl -s http://localhost:3010/api/version || true
232 exit 0
233 fi
234 sleep 2
235 done
236 echo "FAIL: /healthz did not return 200 after 6s"
237 systemctl status gluecron --no-pager | head -10 || true
238 journalctl -u gluecron -n 30 --no-pager || true
239 exit 1
240
241 - name: Notify step — smoke-test (succeeded)
242 if: success() && env.DEPLOY_EVENT_TOKEN != ''
243 uses: ./.github/actions/notify-deploy-step
244 with:
245 step_name: smoke-test
246 status: succeeded
247 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
248 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
249
250 - name: Notify step — smoke-test (failed)
251 if: failure() && steps.smoke.conclusion == 'failure' && env.DEPLOY_EVENT_TOKEN != ''
252 uses: ./.github/actions/notify-deploy-step
253 with:
254 step_name: smoke-test
255 status: failed
256 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
257 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
258
259 # ─── 3b. Full post-deploy smoke suite (Block S1+S3) ──────────────────
260 # `/healthz` alone is NOT enough — it doesn't touch the DB schema, so
261 # a broken migration leaves it green while every real page crashes
262 # selecting columns that don't exist. The post-deploy-smoke script
263 # hits 15 critical endpoints (login renders, /api/version, /demo,
264 # /mcp, /sw.js, etc.) and verifies the LATEST drizzle/*.sql is in
265 # the running process's reported migrations list. If ANY check
266 # fails, the workflow auto-rolls back.
267 - name: Notify step — full-smoke (in_progress)
268 if: env.DEPLOY_EVENT_TOKEN != ''
269 uses: ./.github/actions/notify-deploy-step
270 with:
271 step_name: full-smoke
272 status: in_progress
273 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
274 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
275
276 - name: Full post-deploy smoke suite
277 id: full_smoke
278 uses: appleboy/ssh-action@v1.2.0
279 with:
280 host: ${{ secrets.HETZNER_HOST }}
281 username: ${{ secrets.HETZNER_USER }}
282 key: ${{ secrets.HETZNER_SSH_KEY }}
283 script_stop: true
284 script: |
285 set -e
286 cd /opt/gluecron
287 BUN=/root/.bun/bin/bun
288 export GLUECRON_HOST="http://localhost:3010"
289 echo "==> running 15-endpoint smoke suite against $GLUECRON_HOST"
290 # The script exits 1 on endpoint failure, 2 on missing
291 # migration. We treat both as fatal so the workflow rolls
292 # back. stdout/stderr stream live to the GH Actions log.
293 "$BUN" run scripts/post-deploy-smoke.ts
294
295 - name: Notify step — full-smoke (succeeded)
296 if: success() && env.DEPLOY_EVENT_TOKEN != ''
297 uses: ./.github/actions/notify-deploy-step
298 with:
299 step_name: full-smoke
300 status: succeeded
301 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
302 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
303
304 - name: Notify step — full-smoke (failed)
305 if: failure() && steps.full_smoke.conclusion == 'failure' && env.DEPLOY_EVENT_TOKEN != ''
306 uses: ./.github/actions/notify-deploy-step
307 with:
308 step_name: full-smoke
309 status: failed
310 app_base_url: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
311 deploy_event_token: ${{ secrets.DEPLOY_EVENT_TOKEN }}
312
313 # ─── 4. Auto-rollback on smoke failure ──────────────────────────────
314 # Only rolls back if the workflow was triggered by a normal push.
315 # Manual workflow_dispatch runs SKIP rollback so the operator can
316 # diagnose the new code on the box before reverting. This stops the
317 # pathological case where rollback masks the real failure by reverting
318 # to an already-broken previous SHA.
319 #
320 # S1 (2026-05-14): rollback now also fires when the FULL smoke suite
321 # fails (steps.full_smoke), not just the basic /healthz curl. Recursion
322 # cap: only ONE rollback attempt per workflow run (the `if:` guard
323 # naturally enforces this — the only rollback step in the file).
324 - name: Rollback on failure
325 id: rollback
326 if: failure() && github.event_name == 'push' && (steps.smoke.conclusion == 'failure' || steps.full_smoke.conclusion == 'failure')
327 uses: appleboy/ssh-action@v1.2.0
328 with:
329 host: ${{ secrets.HETZNER_HOST }}
330 username: ${{ secrets.HETZNER_USER }}
331 key: ${{ secrets.HETZNER_SSH_KEY }}
332 script_stop: false
333 script: |
334 cd /opt/gluecron
335 prev=$(cat /tmp/gluecron_prev_sha 2>/dev/null || true)
336 if [ -z "$prev" ]; then
337 echo "ROLLBACK SKIPPED: no /tmp/gluecron_prev_sha — human intervention required"
338 exit 1
339 fi
340 echo "ROLLED BACK to $prev because post-deploy smoke failed"
341 git reset --hard "$prev"
342 # Don't re-run migrations here: rolling back schema is
343 # destructive and migrations are forward-only. We just put
344 # the code back to where it was and restart. If a migration
345 # is the reason the new code is incompatible with the old,
346 # the operator must intervene manually.
347 BUN=/root/.bun/bin/bun
348 if [ -f bun.lock ] && [ -d node_modules ]; then
349 echo "==> reusing existing node_modules (lockfile hash check skipped during rollback)"
350 else
351 # Drop `|| true` (AUDIT-v2.md P0 #6): if rollback bun install
352 # fails, the service must NOT be restarted against half-installed
353 # node_modules. Fail loudly so the operator can intervene.
354 "$BUN" install --frozen-lockfile
355 fi
356 systemctl restart gluecron
357 # Verify the rollback target itself comes up green.
358 sleep 3
359 for i in 1 2 3; do
360 code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3010/healthz)
361 echo "Rollback healthz attempt $i: $code"
362 if [ "$code" = "200" ]; then
363 echo "OK: rolled-back instance is healthy"
364 exit 0
365 fi
366 sleep 2
367 done
368 echo "WARN: rollback target ALSO failed /healthz — human intervention required"
369 systemctl status gluecron --no-pager | head -20 || true
370 exit 1
371
372 # ─── 5. Failure diagnostics — captured into a file for summary + AI ──
373 - name: Capture failure context
374 if: failure()
375 id: ctx
376 uses: appleboy/ssh-action@v1.2.0
377 with:
378 host: ${{ secrets.HETZNER_HOST }}
379 username: ${{ secrets.HETZNER_USER }}
380 key: ${{ secrets.HETZNER_SSH_KEY }}
381 script_stop: false
382 script: |
383 echo "===== systemd status ====="
384 systemctl status gluecron --no-pager 2>&1 | head -40 || true
385 echo ""
386 echo "===== last 80 journal lines (gluecron) ====="
387 journalctl -u gluecron -n 80 --no-pager --output=cat 2>&1 || true
388 echo ""
389 echo "===== caddy validate ====="
390 caddy validate --config /etc/caddy/Caddyfile 2>&1 | head -20 || true
391 echo ""
392 echo "===== /healthz from inside box ====="
393 curl -s -w "\nHTTP %{http_code}\n" http://localhost:3010/healthz 2>&1 || true
394 echo ""
395 echo "===== port 3010 listener ====="
396 # The service runs on 3010 (see line 240, 301, 370). Diagnostics
397 # were curling 3000 for months, which always failed and made
398 # every failure dump useless. (AUDIT-v2.md P0 #5.)
399 ss -tlnp 2>&1 | grep ':3010' || echo '(nothing listening on :3010)'
400
401 # Always post the captured diagnostics to the workflow summary so the
402 # owner can read what broke without SSH'ing or grepping log files.
403 - name: Post diagnostics to summary
404 if: failure() && steps.ctx.outputs.stdout != ''
405 env:
406 DIAG: ${{ steps.ctx.outputs.stdout }}
407 run: |
408 {
409 echo "## ❌ Deploy failed — diagnostics"
410 echo ""
411 echo "**Commit:** \`${GITHUB_SHA:0:7}\` — ${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-${GITHUB_SHA:0:7}}"
412 echo "**Run:** [#${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
413 echo ""
414 echo '```'
415 echo "$DIAG"
416 echo '```'
417 } >> $GITHUB_STEP_SUMMARY
418
419 - name: AI root-cause analysis (Claude)
420 if: failure() && env.ANTHROPIC_API_KEY != ''
421 env:
422 ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
423 DEPLOY_LOGS: ${{ steps.ctx.outputs.stdout }}
424 COMMIT_SHA: ${{ github.sha }}
425 COMMIT_MSG: ${{ github.event.head_commit.message }}
426 run: |
427 set +e
428 # Build the prompt. Use python3 to JSON-encode every untrusted input
429 # (commit message, deploy logs, SHA) so backticks, $(…), quotes, or
430 # newlines in a commit message cannot break out of the JSON string
431 # or execute on the runner. The previous unquoted heredoc allowed
432 # arbitrary command execution via crafted commit messages.
433 python3 - <<'PY' > /tmp/prompt.json
434 import json, os
435 payload = {
436 "model": "claude-haiku-4-5-20251001",
437 "max_tokens": 600,
438 "system": "You are a senior SRE diagnosing a failed deploy. Read the systemd status, journal, and curl output. In 1 short paragraph (under 100 words), identify the most likely root cause and the single fastest fix. Be direct, no preamble.",
439 "messages": [{
440 "role": "user",
441 "content": "Commit: " + os.environ.get("COMMIT_SHA", "") +
442 "\nMessage: " + os.environ.get("COMMIT_MSG", "") +
443 "\n\nDeploy logs:\n" + os.environ.get("DEPLOY_LOGS", ""),
444 }],
445 }
446 print(json.dumps(payload))
447 PY
448 response=$(curl -s https://api.anthropic.com/v1/messages \
449 -H "x-api-key: $ANTHROPIC_API_KEY" \
450 -H "anthropic-version: 2023-06-01" \
451 -H "content-type: application/json" \
452 --data @/tmp/prompt.json)
453 analysis=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('content',[{}])[0].get('text','(no analysis)'))" 2>/dev/null)
454 echo "## 🤖 AI Failure Analysis" >> $GITHUB_STEP_SUMMARY
455 echo "" >> $GITHUB_STEP_SUMMARY
456 echo "$analysis" >> $GITHUB_STEP_SUMMARY
457 echo "" >> $GITHUB_STEP_SUMMARY
458 echo "**Commit:** \`${COMMIT_SHA:0:7}\` — $COMMIT_MSG" >> $GITHUB_STEP_SUMMARY
459
460 # ─── 6. Optional webhook notification ───────────────────────────────
461 - name: Notify webhook
462 if: always() && env.DEPLOY_WEBHOOK_URL != ''
463 env:
464 DEPLOY_WEBHOOK_URL: ${{ secrets.DEPLOY_WEBHOOK_URL }}
465 STATUS: ${{ job.status }}
466 run: |
467 curl -s -X POST "$DEPLOY_WEBHOOK_URL" \
468 -H "content-type: application/json" \
469 --data "{\"status\":\"$STATUS\",\"target\":\"gluecron.com\",\"sha\":\"${{ github.sha }}\",\"run\":\"${{ github.run_id }}\"}" || true
470
471 # ─── 7. Workflow summary on success ─────────────────────────────────
472 - name: Success summary
473 if: success()
474 run: |
475 echo "## ✅ Deploy succeeded" >> $GITHUB_STEP_SUMMARY
476 echo "" >> $GITHUB_STEP_SUMMARY
477 echo "- **Target:** https://gluecron.com" >> $GITHUB_STEP_SUMMARY
478 echo "- **SHA:** \`${GITHUB_SHA:0:7}\`" >> $GITHUB_STEP_SUMMARY
479 echo "- **Health:** /healthz → 200" >> $GITHUB_STEP_SUMMARY
480
481 # ─── 8. Block N3 — POST deploy-finished event to the live site ───────
482 # The site's admin status pill flips to "Deployed Ns ago" or
483 # "Deploy failed Nm ago" the instant this lands. `if: always()` so we
484 # always report final state (including failure); the inner `if:` flag
485 # splits success vs failure for the payload body. We never `--fail` —
486 # a 5xx must not retroactively break a deploy that actually succeeded.
487 - name: Notify deploy finished (success)
488 if: success() && env.DEPLOY_EVENT_TOKEN != ''
489 env:
490 DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
491 APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
492 START_EPOCH: ${{ steps.start.outputs.epoch }}
493 run: |
494 DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
495 curl --silent --show-error --max-time 10 \
496 -X POST "$APP_BASE_URL/api/events/deploy/finished" \
497 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
498 -H "content-type: application/json" \
499 --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"succeeded\",\"duration_ms\":$DUR_MS}" \
500 || echo "(deploy-finished[succeeded] notify failed — continuing)"
501
502 - name: Notify deploy finished (failure)
503 if: failure() && env.DEPLOY_EVENT_TOKEN != ''
504 env:
505 DEPLOY_EVENT_TOKEN: ${{ secrets.DEPLOY_EVENT_TOKEN }}
506 APP_BASE_URL: ${{ secrets.APP_BASE_URL || 'https://gluecron.com' }}
507 START_EPOCH: ${{ steps.start.outputs.epoch }}
508 DIAG: ${{ steps.ctx.outputs.stdout }}
509 ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }}
510 SMOKE_OUTCOME: ${{ steps.smoke.conclusion }}
511 FULL_SMOKE_OUTCOME: ${{ steps.full_smoke.conclusion }}
512 run: |
513 DUR_MS=$(( ( $(date +%s) - START_EPOCH ) * 1000 ))
514 # Build a short reason header. S1 (2026-05-14): the deploy-
515 # finished payload now records WHICH smoke layer failed and
516 # whether the rollback succeeded, so /admin/deploys shows a
517 # red pill with the actual cause instead of "deploy failed".
518 REASON_HEADER=""
519 if [ "$FULL_SMOKE_OUTCOME" = "failure" ]; then
520 REASON_HEADER="post-deploy smoke suite failed"
521 elif [ "$SMOKE_OUTCOME" = "failure" ]; then
522 REASON_HEADER="/healthz smoke failed"
523 fi
524 if [ -n "$ROLLBACK_OUTCOME" ] && [ "$ROLLBACK_OUTCOME" != "skipped" ]; then
525 if [ "$ROLLBACK_OUTCOME" = "success" ]; then
526 REASON_HEADER="ROLLED BACK — $REASON_HEADER"
527 else
528 REASON_HEADER="ROLLBACK FAILED — $REASON_HEADER — human intervention required"
529 fi
530 fi
531 # First 1 KB of diagnostics — keeps the JSON small and the DB row sane.
532 ERR_TEXT=$(printf '%s\n\n%s' "${REASON_HEADER:-deploy failed}" "${DIAG:-see workflow logs}" | head -c 1024)
533 # jq -Rs '.' is the safest way to JSON-escape arbitrary multi-line text.
534 if command -v jq >/dev/null 2>&1; then
535 ERR_JSON=$(printf '%s' "$ERR_TEXT" | jq -Rs '.')
536 else
537 ERR_JSON=$(printf '%s' "$ERR_TEXT" | python3 -c "import sys,json;print(json.dumps(sys.stdin.read()))")
538 fi
539 curl --silent --show-error --max-time 10 \
540 -X POST "$APP_BASE_URL/api/events/deploy/finished" \
541 -H "authorization: Bearer $DEPLOY_EVENT_TOKEN" \
542 -H "content-type: application/json" \
543 --data "{\"run_id\":\"${{ github.run_id }}\",\"sha\":\"${{ github.sha }}\",\"status\":\"failed\",\"duration_ms\":$DUR_MS,\"error\":$ERR_JSON}" \
544 || echo "(deploy-finished[failed] notify failed — continuing)"
Modifiedsrc/lib/autopilot.ts+15−5View fileUnifiedSplit
@@ -312,6 +312,9 @@ function advancementScanDayOfWeek(): number {
312312const SMART_DIGEST_INTERVAL_MS = 22 * 60 * 60 * 1000; // 22h between outer checks
313313let _lastSmartDigestAt = 0;
314314
315/** Hourly rate limit for the zero-mirrors warning (see mirror-sync task). */
316let lastZeroMirrorWarnAt = 0;
317
315318/**
316319 * Default task set. Each task is a thin wrapper around an existing locked
317320 * helper — no gate/merge logic is duplicated here.
@@ -349,11 +352,18 @@ export function defaultTasks(): AutopilotTask[] {
349352 // Not a throw: zero-due is legitimate on a host with no mirrors.
350353 // But it is ALSO what a wiped/never-created mirror table looks
351354 // like, and that ambiguity is what let this run silently for weeks.
352 // Say so, every tick, so "nothing to do" can never again be read as
353 // "everything is mirrored".
354 console.warn(
355 "[autopilot] mirror-sync: NO mirrors are enabled and due — this tick mirrored nothing. If repos are expected to mirror, none is configured."
356 );
355 // Say so — but hourly, not every 5-minute tick: 288 identical
356 // warnings a day teach operators to filter the exact line that
357 // matters (the Vapron instance flagged it as pure noise 2026-08-22).
358 // The signal is "this host mirrors nothing", which does not change
359 // tick to tick.
360 const now = Date.now();
361 if (now - lastZeroMirrorWarnAt > 60 * 60_000) {
362 lastZeroMirrorWarnAt = now;
363 console.warn(
364 "[autopilot] mirror-sync: NO mirrors are enabled and due — nothing is being mirrored on this host. If repos are expected to mirror, none is configured. (Repeats hourly while true.)"
365 );
366 }
357367 }
358368 },
359369 },
360370
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts