fix(autopilot): mirror-sync could never report failure, and mirroring was dead #5482
3 changed files+136−3
Addedsrc/__tests__/autopilot-mirror-sync-failure.test.ts+89−0View fileUnifiedSplit
@@ -0,0 +1,89 @@
1/**
2 * mirror-sync has to be visible when mirroring is not happening.
3 *
4 * The sibling test (autopilot-task-failure.test.ts) pins the rule that a task
5 * must let its error reach `runAutopilotTick`, after 23 tasks were found
6 * swallowing throws and reporting "23/23 ok" forever.
7 *
8 * mirror-sync escaped that fix, because it never threw in the first place. It
9 * was `run: async () => { await syncAllDue(); }` — and `syncAllDue()` resolves
10 * normally no matter what happens, returning `{total, ok, failed}` counts that
11 * the task discarded. `runMirrorSync` records a failed run and, by its own
12 * comment, "never throws out". So:
13 *
14 * - every mirror failing -> {failed: N} -> discarded -> tick ok
15 * - zero mirrors existing -> {total: 0} -> discarded -> tick ok
16 * - the mirror table unreadable -> listDueMirrors caught and returned []
17 * -> {total: 0} -> tick ok
18 *
19 * Three independent layers, each converting failure into silence.
20 *
21 * What it cost, found 2026-08-11: GitHub->Gluecron mirroring was dead across
22 * the whole estate — vapron 522 commits behind, davenroe 65, alecrae.com 17,
23 * zoobicon.com and jarvis-platform 5 each — while every autopilot tick logged
24 * "mirror-sync ok" and the public status page read "All systems operational".
25 * The drift was found by comparing `git ls-remote` between hosts by hand.
26 *
27 * These tests exercise the REAL task out of `defaultTasks()`, so they fail if
28 * anyone reverts to discarding the result.
29 */
30
31import { describe, expect, mock, test } from "bun:test";
32
33// Must be registered before `../lib/autopilot` is imported, since it binds
34// `syncAllDue` at module load.
35let nextResult: { total: number; ok: number; failed: number } = {
36 total: 0,
37 ok: 0,
38 failed: 0,
39};
40
41mock.module("../lib/mirrors", () => ({
42 syncAllDue: async () => nextResult,
43}));
44
45const { defaultTasks } = await import("../lib/autopilot");
46
47function mirrorSyncTask() {
48 const t = defaultTasks().find((x) => x.name === "mirror-sync");
49 if (!t) throw new Error("mirror-sync task is missing from defaultTasks()");
50 return t;
51}
52
53describe("autopilot mirror-sync", () => {
54 test("throws when any mirror failed, so the tick records ok:false", async () => {
55 nextResult = { total: 3, ok: 0, failed: 3 };
56 await expect(mirrorSyncTask().run()).rejects.toThrow(/3\/3/);
57 });
58
59 test("throws even when only some mirrors failed", async () => {
60 // A partial failure is still mirroring that did not happen. Reporting the
61 // tick as ok because most succeeded is how a single repo drifts unnoticed.
62 nextResult = { total: 5, ok: 4, failed: 1 };
63 await expect(mirrorSyncTask().run()).rejects.toThrow(/1\/5/);
64 });
65
66 test("does not throw when every mirror succeeded", async () => {
67 nextResult = { total: 4, ok: 4, failed: 0 };
68 await expect(mirrorSyncTask().run()).resolves.toBeUndefined();
69 });
70
71 test("zero mirrors due resolves, but warns rather than passing silently", async () => {
72 // Zero-due is legitimate on a host with no mirrors, so it must not throw.
73 // But it is ALSO exactly what a wiped or never-created mirror table looks
74 // like, and that ambiguity is what let this run dead for weeks. The warning
75 // is the only thing distinguishing "nothing to do" from "nothing works".
76 nextResult = { total: 0, ok: 0, failed: 0 };
77 const warnings: string[] = [];
78 const realWarn = console.warn;
79 console.warn = (...args: unknown[]) => {
80 warnings.push(args.map(String).join(" "));
81 };
82 try {
83 await expect(mirrorSyncTask().run()).resolves.toBeUndefined();
84 } finally {
85 console.warn = realWarn;
86 }
87 expect(warnings.join(" ")).toMatch(/NO mirrors/i);
88 });
89});
Modifiedsrc/lib/autopilot.ts+34−1View fileUnifiedSplit
@@ -321,7 +321,40 @@ export function defaultTasks(): AutopilotTask[] {
321321 {
322322 name: "mirror-sync",
323323 run: async () => {
324 await syncAllDue();
324 // `syncAllDue()` resolves normally even when EVERY mirror failed:
325 // `runMirrorSync` records the run as failed and, by its own comment,
326 // "never throws out". So discarding the return value made this task
327 // report ok on every tick regardless of what happened.
328 //
329 // That is exactly the defect this file's header describes for the 23
330 // catch-without-rethrow tasks, in a different shape — a discarded
331 // RESULT rather than a swallowed THROW. The rethrow fix applied there
332 // could not catch this one, because nothing here ever threw.
333 //
334 // What it cost, found 2026-08-11: GitHub→Gluecron mirroring was dead
335 // for every repo — vapron 522 commits behind, davenroe 65, alecrae 17,
336 // zoobicon and jarvis-platform 5 each — while /status reported
337 // "mirror-sync ok" on every tick for weeks, and the public status page
338 // read "All systems operational".
339 const s = await syncAllDue();
340 console.log(
341 `[autopilot] mirror-sync: total=${s.total} ok=${s.ok} failed=${s.failed}`
342 );
343 if (s.failed > 0) {
344 throw new Error(
345 `mirror-sync: ${s.failed}/${s.total} mirror(s) failed — see repo_mirror_runs for per-mirror detail`
346 );
347 }
348 if (s.total === 0) {
349 // Not a throw: zero-due is legitimate on a host with no mirrors.
350 // But it is ALSO what a wiped/never-created mirror table looks
351 // like, and that ambiguity is what let this run silently for weeks.
352 // Say so, every tick, so "nothing to do" can never again be read as
353 // "everything is mirrored".
354 console.warn(
355 "[autopilot] mirror-sync: NO mirrors are enabled and due — this tick mirrored nothing. If repos are expected to mirror, none is configured."
356 );
357 }
325358 },
326359 },
327360 {
Modifiedsrc/lib/mirrors.ts+13−2View fileUnifiedSplit
@@ -404,8 +404,19 @@ export async function listDueMirrors(
404404 }
405405 }
406406 return due;
407 } catch {
408 return [];
407 } catch (err) {
408 // Do NOT report a database failure as "nothing is due".
409 //
410 // An empty array is indistinguishable from success to every caller, so a
411 // broken query here made `syncAllDue()` return {total:0, ok:0, failed:0}
412 // and the autopilot record a clean tick. That was one of three layers
413 // that hid GitHub→Gluecron mirroring being dead for weeks (2026-08-11).
414 //
415 // The only caller is `syncAllDue`, reached from the mirror-sync autopilot
416 // task, which now surfaces a throw as a failed tick — which is precisely
417 // what an unreadable mirror table should produce.
418 console.error("[mirrors] listDueMirrors failed — cannot determine what is due:", err);
419 throw err instanceof Error ? err : new Error(String(err));
409420 }
410421}
411422
412423
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts