chore(ops): gluecron-failure-notify@ unit — every timer/service fails loudly #5522
1 changed file+24−0
Addedscripts/systemd/gluecron-failure-notify@.service+24−0View fileUnifiedSplit
@@ -0,0 +1,24 @@
1# Vapron spine-parity ask #2 (2026-08-23) — every unit fails loudly.
2# Modeled on Vapron's vapron-failure-notify@ pattern, born of their
3# crash-looped mail receiver that stayed dead two weeks unnoticed.
4#
5# Install (as root on the box):
6# cp scripts/systemd/gluecron-failure-notify@.service /etc/systemd/system/
7# For each unit that must page on failure, add a drop-in:
8# systemctl edit gluecron-update.service # and backup, restore-drill
9# containing:
10# [Unit]
11# OnFailure=gluecron-failure-notify@%n.service
12# then: systemctl daemon-reload
13#
14# Pages via MONITOR_ALERT_WEBHOOK_URL from /opt/gluecron/.env — the same
15# Slack/ntfy endpoint the app's spine fanout uses, so one URL configures
16# both app-level and unit-level paging. Silent no-op when unset.
17
18[Unit]
19Description=Gluecron failure notifier for %i
20
21[Service]
22Type=oneshot
23EnvironmentFile=-/opt/gluecron/.env
24ExecStart=/bin/bash -c 'if [ -n "${MONITOR_ALERT_WEBHOOK_URL:-}" ]; then curl -fsS -m 10 -H "Content-Type: application/json" -d "{\"text\":\"[critical] systemd unit %i FAILED on gluecron.com — journalctl -u %i -n 50\",\"title\":\"unit %i failed\",\"severity\":\"critical\",\"source\":\"gluecron-161\",\"kind\":\"unit_failed\"}" "$MONITOR_ALERT_WEBHOOK_URL" || true; else echo "MONITOR_ALERT_WEBHOOK_URL unset — unit failure for %i not paged"; fi'
025
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts