fix(nav-audit): the error detector flagged any number between 400 and 499 #5498
2 changed files+145−5
Modifiedscripts/nav-audit.ts+48−5View fileUnifiedSplit
@@ -66,14 +66,52 @@ function authHeaders(url: string): Record<string, string> {
6666 return {};
6767}
6868
69// A 200 that renders an error page still counts as broken. Sniff the title +
70// obvious error markers.
71function errorBody(html: string): string {
69/**
70 * A 200 that renders an error page still counts as broken — but only an
71 * ACTUAL error page does.
72 *
73 * The previous body test was `/>\s*4\d\d\s*</`, which matches any three-digit
74 * number from 400-499 rendered between tags. That is not an error signature,
75 * it is arithmetic. On 2026-08-12 it failed seven live pages that were
76 * completely healthy:
77 *
78 * /contributors → <span class="contrib-num">446</span> commits
79 * /coupling → <span ...>487</span>
80 *
81 * Both returned 200 with correct titles and full content. The audit reported
82 * them as broken for the crime of having a contributor with 446 commits, and
83 * the number would silently stop matching the moment they reached 500.
84 *
85 * A genuine error page identifies itself structurally, in both the JSX and
86 * string-template branches of `src/views/error-page.tsx`:
87 *
88 * role="main" aria-labelledby="error-page-title"
89 * <h1 id="error-page-title" class="err-title">
90 * data-error-code="404" (string branch)
91 *
92 * Keying on that ARIA relationship is intent-based: it matches what the page
93 * IS, survives class renames and copy edits, and cannot be produced by
94 * rendering an integer. Note also that every real error page already carries a
95 * non-200 status AND an error title, so the digit test never caught anything
96 * the two checks above it missed — it only ever added false positives.
97 *
98 * Exported for `src/__tests__/nav-audit-error-detection.test.ts`. The auditor
99 * had no tests at all, which is why a detector this loose went unnoticed while
100 * everyone read its output as fact.
101 */
102export function errorBody(html: string): string {
72103 const title = (html.match(/<title>([^<]*)<\/title>/i)?.[1] || "").trim();
73104 if (/(^|\s)(not found|error|forbidden|something went wrong)(\s|$)/i.test(title)) {
74105 return `error title: "${title}"`;
75106 }
76 if (/>\s*4\d\d\s*<|Page not found|Internal Server Error|Application error/i.test(html)) {
107 // Structural self-identification — the error page says it is one.
108 if (/aria-labelledby=["']error-page-title["']/i.test(html)) {
109 const code = html.match(/data-error-code=["']([^"']+)["']/i)?.[1];
110 return code ? `error page rendered (code ${code})` : "error page rendered";
111 }
112 // Literal server-error strings that appear without our own error template
113 // (upstream proxy pages, framework defaults).
114 if (/Page not found|Internal Server Error|Application error/i.test(html)) {
77115 return "error content in body";
78116 }
79117 return "";
@@ -319,4 +357,9 @@ async function main() {
319357 process.exit(report(rows));
320358}
321359
322main();
360// Guarded so the module can be imported by tests. `main()` ends in
361// `process.exit(...)`, so without this an import would terminate the test run
362// — which is the mechanical reason this file had no tests.
363if (import.meta.main) {
364 main();
365}
Addedsrc/__tests__/nav-audit-error-detection.test.ts+97−0View fileUnifiedSplit
@@ -0,0 +1,97 @@
1/**
2 * nav-audit's error detector — it must flag error pages and nothing else.
3 *
4 * The auditor had NO tests. That is how its body check shipped as:
5 *
6 * />\s*4\d\d\s*</
7 *
8 * which matches any three-digit number from 400-499 rendered between tags.
9 * On 2026-08-12 a full run reported 18 failures; SEVEN of them were healthy
10 * pages whose only crime was rendering a number in that range — a contributor
11 * with 446 commits, a coupling score of 487. Both returned HTTP 200 with
12 * correct titles and complete content.
13 *
14 * The failure mode is worse than a wrong count. It is a QA tool that produces
15 * confident false alarms, so its real findings get discounted too — and it
16 * would have "fixed itself" invisibly the moment that contributor reached 500
17 * commits, which is not a fix, it is the bug moving.
18 *
19 * The fixtures below are real markup, copied from live responses, so a future
20 * refactor of the error page or the contributor card has to keep passing
21 * against what is actually served rather than against a paraphrase.
22 */
23
24import { describe, expect, it } from "bun:test";
25import { errorBody } from "../../scripts/nav-audit";
26
27// --- Real markup, captured from https://gluecron.com on 2026-08-12 ----------
28
29/** The false positive: a contributor card on a fully healthy page. */
30const CONTRIBUTORS_HTML = `<!doctype html><html><head><title>Contributors — ccantynz/alecrae.com — gluecron</title></head><body><main><div class="contrib-card-handle">@noreply</div><div class="contrib-meta-row"><span class="contrib-num">446</span><span class="contrib-meta-label">commits</span><span class="sep">·</span><span class="contrib-add contrib-num">+412</span></div></main></body></html>`;
31
32/** The other false positive: a coupling score. */
33const COUPLING_HTML = `<!doctype html><html><head><title>Insights — ccantynz/voxlen — gluecron</title></head><body><main><span class="metric">487</span></main></body></html>`;
34
35/** A genuine error page — the string-template branch of error-page.tsx. */
36const REAL_ERROR_HTML = `<!doctype html><html><head><title>Not Found — gluecron</title></head><body><main class="err-page" role="main" aria-labelledby="error-page-title" data-error-code="404"><div class="err-code gradient-text error-page-code" aria-hidden="true">404</div><h1 id="error-page-title" class="err-title">We can't find that page.</h1></main></body></html>`;
37
38/**
39 * A genuine error page whose <title> was NOT caught by the title test — the
40 * case the body check exists for. Structural markers must still catch it.
41 */
42const SOFT_ERROR_HTML = `<!doctype html><html><head><title>Contributors — ccantynz/alecrae.com — gluecron</title></head><body><main class="err-page" role="main" aria-labelledby="error-page-title" data-error-code="500"><h1 id="error-page-title" class="err-title">Something broke.</h1></main></body></html>`;
43
44describe("errorBody — healthy pages are not flagged", () => {
45 it("does not flag a contributor card showing 446 commits", () => {
46 expect(errorBody(CONTRIBUTORS_HTML)).toBe("");
47 });
48
49 it("does not flag a coupling metric of 487", () => {
50 expect(errorBody(COUPLING_HTML)).toBe("");
51 });
52
53 it("does not flag ANY bare number in the 400-499 range", () => {
54 // The old regex made this range special for no reason. Walk it so a
55 // reintroduction cannot hide in the part of the range nobody sampled.
56 for (const n of [400, 404, 418, 446, 487, 499]) {
57 const html = `<html><head><title>Repo — gluecron</title></head><body><span class="stat">${n}</span></body></html>`;
58 expect(errorBody(html)).toBe("");
59 }
60 });
61});
62
63describe("errorBody — real error pages are flagged", () => {
64 it("flags a 404 via its error title", () => {
65 expect(errorBody(REAL_ERROR_HTML)).toContain("error title");
66 });
67
68 it("flags an error page that slipped past the title check, via ARIA", () => {
69 // This is the case the body check exists for: HTTP 200, innocuous title,
70 // error template rendered. Structure catches what the title missed.
71 const out = errorBody(SOFT_ERROR_HTML);
72 expect(out).not.toBe("");
73 expect(out).toContain("500");
74 });
75
76 it("still flags upstream/framework error strings without our template", () => {
77 const proxy = `<html><head><title>Oops — gluecron</title></head><body>Internal Server Error</body></html>`;
78 expect(errorBody(proxy)).toBe("error content in body");
79 });
80});
81
82describe("errorBody — the locator is intent-based, not cosmetic", () => {
83 it("survives a class rename on the error page", () => {
84 // Intent-based resolution: the ARIA relationship identifies the page, so
85 // restyling must not blind the audit. `err-code`/`err-title` are exactly
86 // the kind of cosmetic hook that gets renamed in a redesign.
87 const restyled = SOFT_ERROR_HTML
88 .replace(/class="err-page"/, 'class="ErrorScreen_root__x7f2"')
89 .replace(/class="err-title"/, 'class="ErrorScreen_h1__9ab3"');
90 expect(errorBody(restyled)).not.toBe("");
91 });
92
93 it("is not fooled by the phrase appearing in ordinary prose", () => {
94 const prose = `<html><head><title>Docs — gluecron</title></head><body><p>Set aria-labelledby on your headings.</p></body></html>`;
95 expect(errorBody(prose)).toBe("");
96 });
97});
098
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts