feat(zero-noise): signal verdicts + published precision on /status #5524
7 changed files+1137−0
Addeddrizzle/0127_signal_verdicts.sql+36−0View fileUnifiedSplit
@@ -0,0 +1,36 @@
1-- signal_verdicts — one standing developer verdict per signal
2-- (2026-08-24, Zero Noise pillar 4: published precision).
3--
4-- A "signal" is anything the platform showed a developer and claimed was
5-- worth their attention. Today the only kind is "ai_review_comment"
6-- (ref_id = pr_comments.id where is_ai_review = true); the kind axis
7-- exists so gate/alert verdicts can join the same table later without a
8-- second migration.
9--
10-- verdict is "acted" (the finding was right — I did something about it)
11-- or "dismissed" (the finding was wrong). One standing verdict per
12-- signal: the unique (signal_kind, ref_id) key makes a new decision an
13-- upsert that overwrites the old one, never a second row.
14--
15-- These rows feed the /status "Signal precision" figure
16-- (src/lib/signal-precision.ts): acted / (acted + dismissed) over a
17-- rolling window, published with its sample size — computed, not
18-- claimed. Display/reporting only: nothing here gates merging.
19--
20-- No FK from ref_id to pr_comments: the kind axis means ref_id will
21-- point at different tables per kind, so referential integrity is
22-- per-kind application logic (the verdict routes verify the comment
23-- exists and is an AI review comment before writing).
24
25--> statement-breakpoint
26CREATE TABLE IF NOT EXISTS "signal_verdicts" (
27 "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
28 "signal_kind" text NOT NULL,
29 "ref_id" uuid NOT NULL,
30 "repository_id" uuid NOT NULL REFERENCES "repositories"("id") ON DELETE CASCADE,
31 "verdict" text NOT NULL,
32 "decided_by" uuid NOT NULL REFERENCES "users"("id"),
33 "decided_at" timestamp with time zone DEFAULT now() NOT NULL
34);
35--> statement-breakpoint
36CREATE UNIQUE INDEX IF NOT EXISTS "signal_verdicts_kind_ref_uq" ON "signal_verdicts" ("signal_kind","ref_id");
Modifiedsrc/__tests__/copy-honesty.test.ts+23−0View fileUnifiedSplit
@@ -100,6 +100,29 @@ describe("copy honesty — rendered public pages", () => {
100100 expect(body).not.toContain("PostgreSQL (pgvector)");
101101 });
102102
103 it("/status: signal precision is computed-not-claimed — honest empty state, rate never without sample size", async () => {
104 const body = await page("/status");
105 // The section renders in every state (no-DB, under-sample, published)
106 // and always carries the provenance sentence.
107 expect(body).toContain("Signal precision");
108 expect(body).toContain(
109 "Computed from developer verdicts on AI review findings, not claimed."
110 );
111
112 const s = src("src/routes/status.tsx");
113 // The honest empty state must exist as a source path: below MIN_SAMPLE
114 // the page names the real judged/total counts and the publication
115 // floor instead of any percentage.
116 expect(s).toContain("Not enough verdicts yet.");
117 expect(s).toContain("signalPrecision.precision === null");
118 expect(s).toContain("Precision publishes at");
119 // A rendered rate always carries its sample size beside it.
120 expect(s).toContain("Precision · {signalPrecision.judged} verdicts");
121 expect(s).toContain("{signalPrecision.judged} verdicts");
122 // And the store-unreachable state says so rather than showing a figure.
123 expect(s).toContain("The verdict store could not be read");
124 });
125
103126 it("/search?type=code: links the real cross-repo search instead of calling it a roadmap item", async () => {
104127 const res = await app.request("/search?q=foo&type=code");
105128 // Anonymous search may 200 or redirect depending on env; only assert
Addedsrc/__tests__/signal-precision.test.ts+472−0View fileUnifiedSplit
@@ -0,0 +1,472 @@
1/**
2 * Signal verdicts + published precision — Zero Noise pillar 4
3 * (docs/VISION-ZERO-NOISE.md, build-map item 1).
4 *
5 * A) Pure precision math (src/lib/signal-precision.ts): null below
6 * MIN_SAMPLE (including judged = 0, so no division-by-zero path),
7 * all-dismissed = 0%, all-acted = 100%, unknown verdict strings
8 * ignored, totalSignals passed through untouched.
9 * B) The AI-review-signal predicate: is_ai_review is the single source
10 * of truth — a human's comment never gets a verdict.
11 * C) canJudgeSignal (pulls.tsx): PR author or write access; narrower
12 * than canResolvePrThread (no thread-author carve-out).
13 * D) Verdict routes: upsert-overwrite semantics of the unique
14 * (signal_kind, ref_id) key, 400 on non-AI comments and bad verdict
15 * values, permission refusal for read-only strangers, clear deletes.
16 *
17 * Pure halves run everywhere; DB/route tests sit behind
18 * `describe.skipIf(!HAS_DB)` — the same pattern as
19 * pr-review-ergonomics.test.ts.
20 */
21
22import { describe, it, expect } from "bun:test";
23import { randomBytes } from "crypto";
24
25import {
26 MIN_SAMPLE,
27 SIGNAL_WINDOW_DAYS,
28 SIGNAL_KIND_AI_REVIEW,
29 isAiReviewSignal,
30 summarisePrecision,
31} from "../lib/signal-precision";
32import { canJudgeSignal } from "../routes/pulls";
33
34const HAS_DB = Boolean(process.env.DATABASE_URL);
35
36// ---------------------------------------------------------------------------
37// A — precision math (pure)
38// ---------------------------------------------------------------------------
39
40const acted = { verdict: "acted" };
41const dismissed = { verdict: "dismissed" };
42
43describe("summarisePrecision", () => {
44 it("MIN_SAMPLE is the published floor and is 10", () => {
45 expect(MIN_SAMPLE).toBe(10);
46 expect(SIGNAL_WINDOW_DAYS).toBe(30);
47 });
48
49 it("zero judgments → precision null, counts zero (no division-by-zero path)", () => {
50 const r = summarisePrecision([], 7);
51 expect(r).toEqual({
52 acted: 0,
53 dismissed: 0,
54 judged: 0,
55 totalSignals: 7,
56 precision: null,
57 });
58 });
59
60 it("one below MIN_SAMPLE → still null, counts real", () => {
61 const rows = [
62 ...Array(6).fill(acted),
63 ...Array(MIN_SAMPLE - 1 - 6).fill(dismissed),
64 ];
65 const r = summarisePrecision(rows, 20);
66 expect(r.judged).toBe(MIN_SAMPLE - 1);
67 expect(r.acted).toBe(6);
68 expect(r.dismissed).toBe(3);
69 expect(r.precision).toBeNull();
70 });
71
72 it("exactly MIN_SAMPLE publishes", () => {
73 const rows = [...Array(7).fill(acted), ...Array(3).fill(dismissed)];
74 const r = summarisePrecision(rows, 12);
75 expect(r.judged).toBe(10);
76 expect(r.precision).toBeCloseTo(0.7);
77 });
78
79 it("all dismissed at sample → 0%, not null and not NaN", () => {
80 const r = summarisePrecision(Array(MIN_SAMPLE).fill(dismissed), 15);
81 expect(r.precision).toBe(0);
82 expect(r.acted).toBe(0);
83 expect(r.dismissed).toBe(MIN_SAMPLE);
84 });
85
86 it("all acted at sample → 100%", () => {
87 const r = summarisePrecision(Array(MIN_SAMPLE).fill(acted), 10);
88 expect(r.precision).toBe(1);
89 });
90
91 it("unknown verdict strings are ignored — they neither publish nor move the rate", () => {
92 const junk = { verdict: "expired_unjudged" };
93 const r = summarisePrecision(
94 [...Array(9).fill(acted), junk, junk, junk],
95 30
96 );
97 // 9 real judgments + 3 unclassifiable rows: still below the floor.
98 expect(r.judged).toBe(9);
99 expect(r.precision).toBeNull();
100
101 const r2 = summarisePrecision(
102 [...Array(8).fill(acted), ...Array(2).fill(dismissed), junk],
103 30
104 );
105 expect(r2.judged).toBe(10);
106 expect(r2.precision).toBeCloseTo(0.8);
107 });
108
109 it("totalSignals is reported as given (the judged-coverage line's denominator)", () => {
110 expect(summarisePrecision([], 0).totalSignals).toBe(0);
111 expect(summarisePrecision([acted], 1).totalSignals).toBe(1);
112 });
113});
114
115// ---------------------------------------------------------------------------
116// B — the AI-review-signal predicate (pure)
117// ---------------------------------------------------------------------------
118
119describe("isAiReviewSignal", () => {
120 it("is_ai_review = true is a signal; a human's comment never is", () => {
121 expect(isAiReviewSignal({ isAiReview: true })).toBe(true);
122 expect(isAiReviewSignal({ isAiReview: false })).toBe(false);
123 });
124
125 it("the signal kind constant matches the migration's vocabulary", () => {
126 expect(SIGNAL_KIND_AI_REVIEW).toBe("ai_review_comment");
127 });
128});
129
130// ---------------------------------------------------------------------------
131// C — who may judge (pure)
132// ---------------------------------------------------------------------------
133
134describe("canJudgeSignal", () => {
135 const prAuthorId = "00000000-0000-4000-8000-00000000000a";
136 const strangerId = "00000000-0000-4000-8000-00000000000c";
137
138 it("anonymous viewers never judge", () => {
139 expect(
140 canJudgeSignal({ viewerId: null, prAuthorId, viewerAccess: "owner" })
141 ).toBe(false);
142 });
143
144 it("the PR author may judge, even with only read access", () => {
145 expect(
146 canJudgeSignal({ viewerId: prAuthorId, prAuthorId, viewerAccess: "read" })
147 ).toBe(true);
148 });
149
150 it("write access (and above) may judge", () => {
151 for (const access of ["write", "admin", "owner"]) {
152 expect(
153 canJudgeSignal({
154 viewerId: strangerId,
155 prAuthorId,
156 viewerAccess: access,
157 })
158 ).toBe(true);
159 }
160 });
161
162 it("a read-only stranger may not judge", () => {
163 for (const access of ["none", "read"]) {
164 expect(
165 canJudgeSignal({
166 viewerId: strangerId,
167 prAuthorId,
168 viewerAccess: access,
169 })
170 ).toBe(false);
171 }
172 });
173});
174
175// ---------------------------------------------------------------------------
176// Shared DB fixtures (mirrors pr-review-ergonomics.test.ts)
177// ---------------------------------------------------------------------------
178
179/** Session-cookie POST headers that satisfy the csrf same-origin check. */
180function postHeaders(token: string): Record<string, string> {
181 return {
182 cookie: `session=${token}`,
183 host: "localhost",
184 origin: "http://localhost",
185 "content-type": "application/x-www-form-urlencoded",
186 };
187}
188
189async function mkUser(prefix: string) {
190 const { db } = await import("../db");
191 const { users } = await import("../db/schema");
192 const stamp = randomBytes(4).toString("hex");
193 const [u] = await db
194 .insert(users)
195 .values({
196 username: `${prefix}-${stamp}`,
197 email: `${prefix}-${stamp}@test.local`,
198 passwordHash: "x",
199 })
200 .returning();
201 return u!;
202}
203
204async function mkSession(userId: string) {
205 const { db } = await import("../db");
206 const { sessions } = await import("../db/schema");
207 const token = `sigv-${randomBytes(12).toString("hex")}`;
208 await db.insert(sessions).values({
209 userId,
210 token,
211 expiresAt: new Date(Date.now() + 10 * 60_000),
212 });
213 return token;
214}
215
216// ---------------------------------------------------------------------------
217// D — verdict routes (DB)
218// ---------------------------------------------------------------------------
219
220describe.skipIf(!HAS_DB)("POST …/comments/:commentId/verdict(+/clear)", () => {
221 async function seed() {
222 const { db } = await import("../db");
223 const { repositories, pullRequests, prComments, signalVerdicts } =
224 await import("../db/schema");
225 const owner = await mkUser("sgo");
226 const stranger = await mkUser("sgs");
227 const stamp = randomBytes(4).toString("hex");
228 const [repo] = await db
229 .insert(repositories)
230 .values({
231 name: `sg-${stamp}`,
232 ownerId: owner.id,
233 diskPath: `/tmp/sg-${stamp}.git`,
234 defaultBranch: "main",
235 })
236 .returning();
237 const [pr] = await db
238 .insert(pullRequests)
239 .values({
240 repositoryId: repo!.id,
241 authorId: owner.id,
242 title: "Signal-verdict fixture PR",
243 body: "",
244 baseBranch: "main",
245 headBranch: "feature",
246 state: "open",
247 })
248 .returning();
249 const [aiComment] = await db
250 .insert(prComments)
251 .values({
252 pullRequestId: pr!.id,
253 authorId: owner.id,
254 body: "AI finding: possible auth bypass at line 42.",
255 isAiReview: true,
256 filePath: "src/a.ts",
257 lineNumber: 42,
258 })
259 .returning();
260 const [humanComment] = await db
261 .insert(prComments)
262 .values({
263 pullRequestId: pr!.id,
264 authorId: stranger.id,
265 body: "a human comment",
266 })
267 .returning();
268 return {
269 db,
270 signalVerdicts,
271 owner,
272 stranger,
273 repo: repo!,
274 pr: pr!,
275 aiComment: aiComment!,
276 humanComment: humanComment!,
277 };
278 }
279
280 async function verdictRows(
281 fx: Awaited<ReturnType<typeof seed>>,
282 refId: string
283 ) {
284 const { and, eq } = await import("drizzle-orm");
285 return fx.db
286 .select()
287 .from(fx.signalVerdicts)
288 .where(
289 and(
290 eq(fx.signalVerdicts.signalKind, SIGNAL_KIND_AI_REVIEW),
291 eq(fx.signalVerdicts.refId, refId)
292 )
293 );
294 }
295
296 it("PR author judges acted; a second verdict OVERWRITES (one standing row); clear deletes", async () => {
297 const app = (await import("../app")).default;
298 const fx = await seed();
299 const token = await mkSession(fx.owner.id);
300 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
301
302 const res = await app.request(
303 `${base}/comments/${fx.aiComment.id}/verdict`,
304 {
305 method: "POST",
306 headers: postHeaders(token),
307 body: new URLSearchParams({ verdict: "acted" }),
308 }
309 );
310 expect(res.status).toBe(302);
311 expect(res.headers.get("location") || "").not.toContain("error=");
312
313 let rows = await verdictRows(fx, fx.aiComment.id);
314 expect(rows).toHaveLength(1);
315 expect(rows[0]?.verdict).toBe("acted");
316 expect(rows[0]?.decidedBy).toBe(fx.owner.id);
317 expect(rows[0]?.repositoryId).toBe(fx.repo.id);
318
319 // Change of mind: dismissed replaces acted — still exactly one row.
320 const res2 = await app.request(
321 `${base}/comments/${fx.aiComment.id}/verdict`,
322 {
323 method: "POST",
324 headers: postHeaders(token),
325 body: new URLSearchParams({ verdict: "dismissed" }),
326 }
327 );
328 expect(res2.status).toBe(302);
329 rows = await verdictRows(fx, fx.aiComment.id);
330 expect(rows).toHaveLength(1);
331 expect(rows[0]?.verdict).toBe("dismissed");
332
333 // Clear withdraws the standing verdict entirely.
334 const res3 = await app.request(
335 `${base}/comments/${fx.aiComment.id}/verdict/clear`,
336 {
337 method: "POST",
338 headers: postHeaders(token),
339 body: new URLSearchParams({}),
340 }
341 );
342 expect(res3.status).toBe(302);
343 rows = await verdictRows(fx, fx.aiComment.id);
344 expect(rows).toHaveLength(0);
345 });
346
347 it("a human's comment never gets a verdict — 400", async () => {
348 const app = (await import("../app")).default;
349 const fx = await seed();
350 const token = await mkSession(fx.owner.id);
351 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
352
353 const res = await app.request(
354 `${base}/comments/${fx.humanComment.id}/verdict`,
355 {
356 method: "POST",
357 headers: postHeaders(token),
358 body: new URLSearchParams({ verdict: "acted" }),
359 }
360 );
361 expect(res.status).toBe(400);
362 expect(await verdictRows(fx, fx.humanComment.id)).toHaveLength(0);
363 });
364
365 it("a bad verdict value is 400 before any DB write", async () => {
366 const app = (await import("../app")).default;
367 const fx = await seed();
368 const token = await mkSession(fx.owner.id);
369 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
370
371 for (const bad of ["maybe", "", "ACTED"]) {
372 const res = await app.request(
373 `${base}/comments/${fx.aiComment.id}/verdict`,
374 {
375 method: "POST",
376 headers: postHeaders(token),
377 body: new URLSearchParams({ verdict: bad }),
378 }
379 );
380 expect(res.status).toBe(400);
381 }
382 expect(await verdictRows(fx, fx.aiComment.id)).toHaveLength(0);
383 });
384
385 it("a read-only stranger is refused and writes nothing", async () => {
386 const app = (await import("../app")).default;
387 const fx = await seed();
388 const token = await mkSession(fx.stranger.id);
389 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
390
391 const res = await app.request(
392 `${base}/comments/${fx.aiComment.id}/verdict`,
393 {
394 method: "POST",
395 headers: postHeaders(token),
396 body: new URLSearchParams({ verdict: "dismissed" }),
397 }
398 );
399 expect(res.status).toBe(302);
400 expect(res.headers.get("location") || "").toContain("error=");
401 expect(await verdictRows(fx, fx.aiComment.id)).toHaveLength(0);
402 });
403
404 it("anonymous POSTs bounce to login", async () => {
405 const app = (await import("../app")).default;
406 const fx = await seed();
407 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
408 const res = await app.request(
409 `${base}/comments/${fx.aiComment.id}/verdict`,
410 {
411 method: "POST",
412 headers: {
413 host: "localhost",
414 origin: "http://localhost",
415 "content-type": "application/x-www-form-urlencoded",
416 },
417 body: new URLSearchParams({ verdict: "acted" }),
418 }
419 );
420 expect([302, 401]).toContain(res.status);
421 if (res.status === 302) {
422 expect(res.headers.get("location") || "").toContain("/login");
423 }
424 });
425
426 it("the PR page shows the verdict affordances to the author, with the standing choice marked", async () => {
427 const app = (await import("../app")).default;
428 const fx = await seed();
429 const token = await mkSession(fx.owner.id);
430 const base = `/${fx.owner.username}/${fx.repo.name}/pulls/${fx.pr.number}`;
431
432 await app.request(`${base}/comments/${fx.aiComment.id}/verdict`, {
433 method: "POST",
434 headers: postHeaders(token),
435 body: new URLSearchParams({ verdict: "acted" }),
436 });
437
438 const page = await app.request(base, {
439 headers: { cookie: `session=${token}` },
440 });
441 expect(page.status).toBe(200);
442 const html = await page.text();
443 expect(html).toContain("✓ useful");
444 expect(html).toContain("✗ wrong");
445 expect(html).toContain("prs-verdict-btn is-chosen");
446 expect(html).toContain(`/comments/${fx.aiComment.id}/verdict/clear`);
447
448 // Anonymous viewers see the AI comment but no verdict affordances.
449 const anonPage = await app.request(base);
450 if (anonPage.status === 200) {
451 const anonHtml = await anonPage.text();
452 expect(anonHtml).not.toContain("prs-verdict-btn");
453 }
454 });
455
456 it("computeSignalPrecision returns an honest shape from the live tables", async () => {
457 const { computeSignalPrecision } = await import("../lib/signal-precision");
458 const r = await computeSignalPrecision(30);
459 expect(r.acted).toBeGreaterThanOrEqual(0);
460 expect(r.dismissed).toBeGreaterThanOrEqual(0);
461 expect(r.judged).toBe(r.acted + r.dismissed);
462 // Verdicts join back to in-window AI comments, so coverage can never
463 // exceed the signal count.
464 expect(r.judged).toBeLessThanOrEqual(r.totalSignals);
465 if (r.judged < MIN_SAMPLE) {
466 expect(r.precision).toBeNull();
467 } else {
468 expect(r.precision).toBeGreaterThanOrEqual(0);
469 expect(r.precision).toBeLessThanOrEqual(1);
470 }
471 });
472});
Modifiedsrc/db/schema.ts+45−0View fileUnifiedSplit
@@ -5057,3 +5057,48 @@ export const osvScanState = pgTable(
50575057);
50585058
50595059export type OsvScanState = typeof osvScanState.$inferSelect;
5060
5061// ---------------------------------------------------------------------------
5062// Signal verdicts (2026-08-24, migration 0127) — Zero Noise pillar 4.
5063//
5064// One standing developer verdict per signal the platform showed. Today the
5065// only signalKind is "ai_review_comment" (refId = prComments.id where
5066// isAiReview = true); the kind axis exists so gate/alert verdicts can join
5067// this table later without a new migration. verdict is "acted" (the finding
5068// was right) or "dismissed" (the finding was wrong); the unique
5069// (signalKind, refId) key makes every new decision an upsert that
5070// overwrites the old one — never a second row.
5071//
5072// No FK on refId: it points at different tables per kind, so existence +
5073// the AI-review predicate are enforced by the verdict routes (pulls.tsx).
5074// Feeds src/lib/signal-precision.ts → the /status "Signal precision"
5075// figure. Reporting only — never consulted by any merge gate.
5076// ---------------------------------------------------------------------------
5077
5078export const signalVerdicts = pgTable(
5079 "signal_verdicts",
5080 {
5081 id: uuid("id").primaryKey().defaultRandom(),
5082 signalKind: text("signal_kind").notNull(),
5083 refId: uuid("ref_id").notNull(),
5084 repositoryId: uuid("repository_id")
5085 .notNull()
5086 .references(() => repositories.id, { onDelete: "cascade" }),
5087 verdict: text("verdict").notNull(),
5088 decidedBy: uuid("decided_by")
5089 .notNull()
5090 .references(() => users.id),
5091 decidedAt: timestamp("decided_at", { withTimezone: true })
5092 .defaultNow()
5093 .notNull(),
5094 },
5095 (table) => [
5096 uniqueIndex("signal_verdicts_kind_ref_uq").on(
5097 table.signalKind,
5098 table.refId
5099 ),
5100 ]
5101);
5102
5103export type SignalVerdict = typeof signalVerdicts.$inferSelect;
5104export type NewSignalVerdict = typeof signalVerdicts.$inferInsert;
Addedsrc/lib/signal-precision.ts+130−0View fileUnifiedSplit
@@ -0,0 +1,130 @@
1/**
2 * Signal precision — Zero Noise pillar 4 (docs/VISION-ZERO-NOISE.md).
3 *
4 * Publishes the platform's own false-positive rate, computed — never
5 * claimed. A "signal" is anything we showed a developer and asserted was
6 * worth their attention; today the only kind is AI review comments
7 * (pr_comments rows with is_ai_review = true). Developers judge each one
8 * from the PR page ("✓ useful" / "✗ wrong" → signal_verdicts, migration
9 * 0127), and:
10 *
11 * precision = acted / (acted + dismissed) over a rolling window
12 *
13 * Below MIN_SAMPLE judgments the figure is null and the /status surface
14 * says "not enough verdicts yet" — a fabricated rate is exactly the
15 * noise this doctrine exists to kill.
16 *
17 * Split in two so the math is testable without a database:
18 * - summarisePrecision(): pure fold over verdict rows.
19 * - computeSignalPrecision(): the two bounded queries + the fold.
20 */
21
22import { and, eq, gte, sql } from "drizzle-orm";
23import { db } from "../db";
24import { prComments, signalVerdicts } from "../db/schema";
25
26/**
27 * Judgments required before a precision percentage is published. Below
28 * this, `precision` is null and callers must render the honest empty
29 * state with the real judged/total counts.
30 */
31export const MIN_SAMPLE = 10;
32
33/** Rolling window the published figure covers, in days. */
34export const SIGNAL_WINDOW_DAYS = 30;
35
36/** signalKind for AI review comments — the only kind that exists today. */
37export const SIGNAL_KIND_AI_REVIEW = "ai_review_comment";
38
39/**
40 * The AI-review-signal predicate: is this pr_comments row a signal a
41 * developer may pass a verdict on?
42 *
43 * The single source of truth is the is_ai_review column — every AI
44 * surface that posts PR comments (ai-review.ts, ai-review-trio.ts, and
45 * the summary/status comments they emit) sets it, and it is already what
46 * auto-merge, the admin dashboards, and demo-activity mean by "AI review
47 * comment". A human's comment (isAiReview = false) never gets a verdict.
48 */
49export function isAiReviewSignal(c: { isAiReview: boolean }): boolean {
50 return c.isAiReview === true;
51}
52
53export interface SignalPrecision {
54 /** Verdict rows saying the finding was right (developer acted on it). */
55 acted: number;
56 /** Verdict rows saying the finding was wrong. */
57 dismissed: number;
58 /** acted + dismissed — the sample the rate is computed over. */
59 judged: number;
60 /** AI review comments posted in the window (judged or not). */
61 totalSignals: number;
62 /**
63 * acted / judged as a 0..1 fraction, or null when judged < MIN_SAMPLE
64 * — including the judged = 0 case, so no division-by-zero path exists.
65 */
66 precision: number | null;
67}
68
69/**
70 * Pure fold: verdict rows + the window's signal count → the published
71 * figures. Unknown verdict strings are ignored rather than counted into
72 * either bucket — a row we cannot classify must not move the rate.
73 */
74export function summarisePrecision(
75 verdictRows: Array<{ verdict: string }>,
76 totalSignals: number
77): SignalPrecision {
78 let acted = 0;
79 let dismissed = 0;
80 for (const row of verdictRows) {
81 if (row.verdict === "acted") acted++;
82 else if (row.verdict === "dismissed") dismissed++;
83 }
84 const judged = acted + dismissed;
85 return {
86 acted,
87 dismissed,
88 judged,
89 totalSignals,
90 precision: judged >= MIN_SAMPLE ? acted / judged : null,
91 };
92}
93
94/**
95 * The published figure: verdicts on AI review comments posted in the
96 * last `windowDays` days. Both queries are bounded to the window and the
97 * verdict query joins back to pr_comments, so judged can never exceed
98 * totalSignals and a verdict on a signal older than the window ages out
99 * of the rate together with its signal.
100 *
101 * Throws on DB failure — callers that render public surfaces catch and
102 * show an honest "unavailable" state instead of a number.
103 */
104export async function computeSignalPrecision(
105 windowDays: number = SIGNAL_WINDOW_DAYS
106): Promise<SignalPrecision> {
107 const cutoff = new Date(Date.now() - windowDays * 24 * 60 * 60 * 1000);
108
109 const [totalRow] = await db
110 .select({ n: sql<number>`count(*)::int` })
111 .from(prComments)
112 .where(
113 and(eq(prComments.isAiReview, true), gte(prComments.createdAt, cutoff))
114 );
115 const totalSignals = Number(totalRow?.n ?? 0);
116
117 const verdictRows = await db
118 .select({ verdict: signalVerdicts.verdict })
119 .from(signalVerdicts)
120 .innerJoin(prComments, eq(signalVerdicts.refId, prComments.id))
121 .where(
122 and(
123 eq(signalVerdicts.signalKind, SIGNAL_KIND_AI_REVIEW),
124 eq(prComments.isAiReview, true),
125 gte(prComments.createdAt, cutoff)
126 )
127 );
128
129 return summarisePrecision(verdictRows, totalSignals);
130}
Modifiedsrc/routes/pulls.tsx+317−0View fileUnifiedSplit
@@ -29,7 +29,12 @@ import {
2929 repoCollaborators,
3030 pendingReviews,
3131 pendingReviewComments,
32 signalVerdicts,
3233} from "../db/schema";
34import {
35 SIGNAL_KIND_AI_REVIEW,
36 isAiReviewSignal,
37} from "../lib/signal-precision";
3338import { Layout } from "../views/layout";
3439import { RepoHeader, RepoNav, PageHeader, Button as GxButton, Badge as GxBadge, sharedComponentStyles } from "../views/components";
3540import { PendingCommentsBanner } from "../views/pending-comments-banner";
@@ -726,6 +731,53 @@ const PRS_DETAIL_STYLES = `
726731 border-radius: 9999px;
727732 }
728733
734 /* Signal-verdict affordances on AI review comments (Zero Noise pillar 4).
735 Deliberately quiet: small text buttons in the card footer; the chosen
736 side carries the evergreen accent. */
737 .prs-verdict-row {
738 display: flex; align-items: center; gap: 8px;
739 padding: 8px 18px 10px;
740 border-top: 1px dashed var(--border-subtle, var(--border));
741 flex-wrap: wrap;
742 }
743 .prs-verdict-label {
744 font-size: 11.5px;
745 color: var(--text-muted);
746 margin-right: 2px;
747 }
748 .prs-verdict-form { margin: 0; display: inline-flex; }
749 .prs-verdict-btn {
750 padding: 3px 10px;
751 font-size: 12px;
752 font-weight: 600;
753 color: var(--text-muted);
754 background: transparent;
755 border: 1px solid var(--border);
756 border-radius: 9999px;
757 cursor: pointer;
758 transition: border-color 120ms ease, color 120ms ease, background 120ms ease;
759 }
760 .prs-verdict-btn:hover {
761 color: var(--text-strong);
762 border-color: var(--border-strong, var(--border));
763 }
764 .prs-verdict-btn.is-chosen {
765 color: var(--accent);
766 background: color-mix(in srgb, var(--accent) 10%, transparent);
767 border-color: color-mix(in srgb, var(--accent) 45%, transparent);
768 }
769 .prs-verdict-clear {
770 padding: 3px 8px;
771 font-size: 11.5px;
772 color: var(--text-faint, var(--text-muted));
773 background: transparent;
774 border: none;
775 cursor: pointer;
776 text-decoration: underline;
777 text-underline-offset: 2px;
778 }
779 .prs-verdict-clear:hover { color: var(--text-muted); }
780
729781 /* Files-changed link card on conversation tab. (Diff itself is in DiffView.) */
730782 .prs-files-card {
731783 margin-top: 18px;
@@ -2191,6 +2243,25 @@ export function canResolvePrThread(args: {
21912243 return satisfiesAccess(args.viewerAccess as RepoAccessLevel, "write");
21922244}
21932245
2246/**
2247 * Zero Noise pillar 4 — who may pass a verdict on an AI review comment
2248 * (signal_verdicts, migration 0127). Pure so tests can pin the rule: the
2249 * PR author (even with only read access — it's their PR the finding
2250 * landed on) or anyone with write access. Narrower than
2251 * canResolvePrThread on purpose: verdicts feed the published /status
2252 * precision figure, so drive-by readers don't get a vote.
2253 */
2254export function canJudgeSignal(args: {
2255 viewerId: string | null;
2256 prAuthorId: string;
2257 /** RepoAccessLevel string as stashed by requireRepoAccess. */
2258 viewerAccess: string;
2259}): boolean {
2260 if (!args.viewerId) return false;
2261 if (args.viewerId === args.prAuthorId) return true;
2262 return satisfiesAccess(args.viewerAccess as RepoAccessLevel, "write");
2263}
2264
21942265/**
21952266 * Inline (file+line anchored) comments for the diff surfaces, with thread
21962267 * resolve state attached to each thread root. Shared by the PR detail
@@ -4326,6 +4397,37 @@ pulls.get("/:owner/:repo/pulls/:number", softAuth, requireRepoAccess("read"), as
43264397 .map(([anchor]) => anchor)
43274398 );
43284399
4400 // Zero Noise pillar 4 — standing verdicts on this PR's AI review
4401 // comments, plus whether the viewer may judge (PR author or write
4402 // access — canJudgeSignal). One batched lookup; best-effort so a
4403 // missing signal_verdicts table (pre-migration boot) degrades to
4404 // "no verdicts shown", never a dead PR page.
4405 const viewerCanJudge = canJudgeSignal({
4406 viewerId: user?.id ?? null,
4407 prAuthorId: pr.authorId,
4408 viewerAccess: (c.get("repoAccess") as string | undefined) ?? "read",
4409 });
4410 const aiSignalIds = comments
4411 .filter(({ comment }) => isAiReviewSignal(comment))
4412 .map(({ comment }) => comment.id);
4413 const verdictByComment = new Map<string, string>();
4414 if (aiSignalIds.length > 0) {
4415 const verdictRows = await db
4416 .select({
4417 refId: signalVerdicts.refId,
4418 verdict: signalVerdicts.verdict,
4419 })
4420 .from(signalVerdicts)
4421 .where(
4422 and(
4423 eq(signalVerdicts.signalKind, SIGNAL_KIND_AI_REVIEW),
4424 inArray(signalVerdicts.refId, aiSignalIds)
4425 )
4426 )
4427 .catch(() => [] as { refId: string; verdict: string }[]);
4428 for (const r of verdictRows) verdictByComment.set(r.refId, r.verdict);
4429 }
4430
43294431 // Reactions for the PR body + each comment, in parallel.
43304432 const [prReactions, ...prCommentReactions] = await Promise.all([
43314433 summariseReactions("pr", pr.id, user?.id),
@@ -5366,6 +5468,58 @@ pulls.get("/:owner/:repo/pulls/:number", softAuth, requireRepoAccess("read"), as
53665468 )}
53675469 />
53685470 </div>
5471 {/* Zero Noise pillar 4 — one-click verdict on an AI
5472 finding. Judges only (PR author / write access); the
5473 chosen side gets the evergreen accent. Feeds the
5474 published /status "Signal precision" figure. */}
5475 {comment.isAiReview && viewerCanJudge && (
5476 <div class="prs-verdict-row">
5477 <span class="prs-verdict-label">Was this finding right?</span>
5478 <form
5479 method="post"
5480 action={`/${ownerName}/${repoName}/pulls/${pr.number}/comments/${comment.id}/verdict`}
5481 class="prs-verdict-form"
5482 >
5483 <input type="hidden" name="verdict" value="acted" />
5484 <button
5485 type="submit"
5486 class={`prs-verdict-btn${verdictByComment.get(comment.id) === "acted" ? " is-chosen" : ""}`}
5487 title="This finding was correct — I acted on it"
5488 >
5489 {"✓ useful"}
5490 </button>
5491 </form>
5492 <form
5493 method="post"
5494 action={`/${ownerName}/${repoName}/pulls/${pr.number}/comments/${comment.id}/verdict`}
5495 class="prs-verdict-form"
5496 >
5497 <input type="hidden" name="verdict" value="dismissed" />
5498 <button
5499 type="submit"
5500 class={`prs-verdict-btn${verdictByComment.get(comment.id) === "dismissed" ? " is-chosen" : ""}`}
5501 title="This finding was wrong — dismissed"
5502 >
5503 {"✗ wrong"}
5504 </button>
5505 </form>
5506 {verdictByComment.has(comment.id) && (
5507 <form
5508 method="post"
5509 action={`/${ownerName}/${repoName}/pulls/${pr.number}/comments/${comment.id}/verdict/clear`}
5510 class="prs-verdict-form"
5511 >
5512 <button
5513 type="submit"
5514 class="prs-verdict-clear"
5515 title="Withdraw this verdict"
5516 >
5517 clear
5518 </button>
5519 </form>
5520 )}
5521 </div>
5522 )}
53695523 </div>
53705524 );
53715525 })}
@@ -6504,6 +6658,169 @@ pulls.post(
65046658 }
65056659);
65066660
6661// ─── Zero Noise pillar 4 — signal verdicts on AI review comments ───────────
6662// One click from the PR author or anyone with write access records whether
6663// an AI review finding was right ("acted") or wrong ("dismissed") —
6664// signal_verdicts (migration 0127), unique per (kind, comment), so a new
6665// decision overwrites the standing one. These rows feed the published
6666// /status "Signal precision" figure (src/lib/signal-precision.ts).
6667// A human's comment never gets a verdict: non-AI comments 400.
6668// Middleware chain matches the resolve/unresolve siblings above.
6669
6670/** Shared worker for the two verdict routes. Returns null on success, or
6671 * an error whose httpStatus decides between a 4xx response (contract
6672 * violations — the spec's "400 otherwise") and the sibling routes'
6673 * redirect-with-error (permission / not-found). */
6674async function setSignalVerdict(args: {
6675 ownerName: string;
6676 repoName: string;
6677 prNum: number;
6678 commentId: string;
6679 userId: string;
6680 viewerAccess: string;
6681 /** "acted" | "dismissed" to upsert, null to clear. */
6682 verdict: "acted" | "dismissed" | null;
6683}): Promise<{ message: string; httpStatus: 400 | 403 | 404 } | null> {
6684 const repoCtx = await resolveRepo(args.ownerName, args.repoName);
6685 if (!repoCtx) return { message: "Repository not found", httpStatus: 404 };
6686
6687 const [pr] = await db
6688 .select()
6689 .from(pullRequests)
6690 .where(
6691 and(
6692 eq(pullRequests.repositoryId, repoCtx.repo.id),
6693 eq(pullRequests.number, args.prNum)
6694 )
6695 )
6696 .limit(1);
6697 if (!pr) return { message: "Pull request not found", httpStatus: 404 };
6698
6699 const [comment] = await db
6700 .select()
6701 .from(prComments)
6702 .where(
6703 and(
6704 eq(prComments.id, args.commentId),
6705 eq(prComments.pullRequestId, pr.id)
6706 )
6707 )
6708 .limit(1);
6709 if (!comment) return { message: "Comment not found", httpStatus: 404 };
6710 if (!isAiReviewSignal(comment)) {
6711 return {
6712 message: "Verdicts apply only to AI review comments",
6713 httpStatus: 400,
6714 };
6715 }
6716
6717 const allowed = canJudgeSignal({
6718 viewerId: args.userId,
6719 prAuthorId: pr.authorId,
6720 viewerAccess: args.viewerAccess,
6721 });
6722 if (!allowed) {
6723 return {
6724 message: "Only the PR author or someone with write access can judge AI findings",
6725 httpStatus: 403,
6726 };
6727 }
6728
6729 if (args.verdict === null) {
6730 await db
6731 .delete(signalVerdicts)
6732 .where(
6733 and(
6734 eq(signalVerdicts.signalKind, SIGNAL_KIND_AI_REVIEW),
6735 eq(signalVerdicts.refId, comment.id)
6736 )
6737 );
6738 return null;
6739 }
6740
6741 // One standing verdict per signal — a new decision overwrites.
6742 await db
6743 .insert(signalVerdicts)
6744 .values({
6745 signalKind: SIGNAL_KIND_AI_REVIEW,
6746 refId: comment.id,
6747 repositoryId: repoCtx.repo.id,
6748 verdict: args.verdict,
6749 decidedBy: args.userId,
6750 })
6751 .onConflictDoUpdate({
6752 target: [signalVerdicts.signalKind, signalVerdicts.refId],
6753 set: {
6754 verdict: args.verdict,
6755 decidedBy: args.userId,
6756 decidedAt: new Date(),
6757 },
6758 });
6759 return null;
6760}
6761
6762pulls.post(
6763 "/:owner/:repo/pulls/:number/comments/:commentId/verdict",
6764 softAuth,
6765 requireAuth,
6766 requireRepoAccess("read"),
6767 async (c) => {
6768 const { owner: ownerName, repo: repoName } = c.req.param();
6769 const prNum = (parseIdNumber(c.req.param("number")) ?? -1);
6770 const commentId = parseIdUuid(c.req.param("commentId"));
6771 const user = c.get("user")!;
6772 const backUrl = `/${ownerName}/${repoName}/pulls/${prNum}`;
6773 if (!commentId) return c.redirect(backUrl);
6774
6775 const form = await c.req.formData().catch(() => null);
6776 const verdict = form?.get("verdict");
6777 if (verdict !== "acted" && verdict !== "dismissed") {
6778 return c.text('verdict must be "acted" or "dismissed"', 400);
6779 }
6780
6781 const err = await setSignalVerdict({
6782 ownerName,
6783 repoName,
6784 prNum,
6785 commentId,
6786 userId: user.id,
6787 viewerAccess: (c.get("repoAccess") as string | undefined) ?? "read",
6788 verdict,
6789 });
6790 if (err?.httpStatus === 400) return c.text(err.message, 400);
6791 if (err) return c.redirect(`${backUrl}?error=${encodeURIComponent(err.message)}`);
6792 return c.redirect(backUrl);
6793 }
6794);
6795
6796pulls.post(
6797 "/:owner/:repo/pulls/:number/comments/:commentId/verdict/clear",
6798 softAuth,
6799 requireAuth,
6800 requireRepoAccess("read"),
6801 async (c) => {
6802 const { owner: ownerName, repo: repoName } = c.req.param();
6803 const prNum = (parseIdNumber(c.req.param("number")) ?? -1);
6804 const commentId = parseIdUuid(c.req.param("commentId"));
6805 const user = c.get("user")!;
6806 const backUrl = `/${ownerName}/${repoName}/pulls/${prNum}`;
6807 if (!commentId) return c.redirect(backUrl);
6808
6809 const err = await setSignalVerdict({
6810 ownerName,
6811 repoName,
6812 prNum,
6813 commentId,
6814 userId: user.id,
6815 viewerAccess: (c.get("repoAccess") as string | undefined) ?? "read",
6816 verdict: null,
6817 });
6818 if (err?.httpStatus === 400) return c.text(err.message, 400);
6819 if (err) return c.redirect(`${backUrl}?error=${encodeURIComponent(err.message)}`);
6820 return c.redirect(backUrl);
6821 }
6822);
6823
65076824// ─── Batched PR review workflow ─────────────────────────────────────────────
65086825// Reviewers can stage multiple inline comments in a pending_reviews session,
65096826// then submit them all at once as an Approve / Request Changes / Comment review.
Modifiedsrc/routes/status.tsx+114−0View fileUnifiedSplit
@@ -27,6 +27,12 @@ import { getLastTick, getTickCount } from "../lib/autopilot";
2727import { recentRedChecks } from "../lib/synthetic-monitor";
2828import { sendEmail, reportEmailFailure } from "../lib/email";
2929import { config } from "../lib/config";
30import {
31 computeSignalPrecision,
32 MIN_SAMPLE,
33 SIGNAL_WINDOW_DAYS,
34 type SignalPrecision,
35} from "../lib/signal-precision";
3036
3137const status = new Hono<AuthEnv>();
3238
@@ -540,6 +546,20 @@ status.get("/status", async (c) => {
540546 const monitoredServices = services.filter((s) => s.state !== "unknown");
541547 const monitoredUp = monitoredServices.filter((s) => s.state === "ok").length;
542548
549 // Zero Noise pillar 4 — the published AI-review precision figure,
550 // computed from developer verdicts (signal_verdicts, migration 0127) by
551 // src/lib/signal-precision.ts. Three honest states, no fabricated rate:
552 // - null → the verdict store could not be read; say so.
553 // - precision null → fewer than MIN_SAMPLE judgments in the window;
554 // show the real judged/total counts instead.
555 // - a number → the rate, always beside its sample size.
556 let signalPrecision: SignalPrecision | null = null;
557 try {
558 signalPrecision = await computeSignalPrecision(SIGNAL_WINDOW_DAYS);
559 } catch {
560 signalPrecision = null;
561 }
562
543563 return c.html(
544564 <Layout title="Status — gluecron" user={user}>
545565 <style dangerouslySetInnerHTML={{ __html: statusStyles }} />
@@ -649,6 +669,100 @@ status.get("/status", async (c) => {
649669 </div>
650670 )}
651671
672 {/* ─── Signal precision (Zero Noise pillar 4) ─── */}
673 <section class="status-section" aria-labelledby="status-sig-h">
674 <header class="status-section-head">
675 <div>
676 <p class="status-section-eyebrow">Signal quality</p>
677 <h2 class="status-section-title" id="status-sig-h">
678 Signal precision
679 </h2>
680 <p class="status-section-sub">
681 How often the developers who received an AI review finding
682 judged it correct — rolling {SIGNAL_WINDOW_DAYS}-day window.
683 </p>
684 </div>
685 {signalPrecision === null ? (
686 <span class="status-count-pill">Unavailable</span>
687 ) : signalPrecision.precision === null ? (
688 <span class="status-count-pill">Collecting verdicts</span>
689 ) : (
690 <span class="status-count-pill is-ok">
691 {(signalPrecision.precision * 100).toFixed(1)}% ·{" "}
692 {signalPrecision.judged} verdicts
693 </span>
694 )}
695 </header>
696 <div class="status-section-body">
697 {signalPrecision === null ? (
698 <div class="status-empty">
699 <div class="status-empty-orb" aria-hidden="true" />
700 <div class="status-empty-inner">
701 <p class="status-empty-title">
702 Signal precision is unavailable right now.
703 </p>
704 <p class="status-empty-sub">
705 The verdict store could not be read on this request, so no
706 figure is shown — a number we cannot compute is a number we
707 do not publish.
708 </p>
709 </div>
710 </div>
711 ) : signalPrecision.precision === null ? (
712 <div class="status-empty">
713 <div class="status-empty-orb" aria-hidden="true" />
714 <div class="status-empty-inner">
715 <p class="status-empty-title">Not enough verdicts yet.</p>
716 <p class="status-empty-sub">
717 {signalPrecision.judged} of {signalPrecision.totalSignals}{" "}
718 AI review finding
719 {signalPrecision.totalSignals === 1 ? "" : "s"} judged in
720 the last {SIGNAL_WINDOW_DAYS} days. Precision publishes at{" "}
721 {MIN_SAMPLE} judgments — never before, and never as a
722 fabricated rate.
723 </p>
724 </div>
725 </div>
726 ) : (
727 <div class="status-stats-grid">
728 <div class="status-stat">
729 <div class="status-stat-num">
730 {(signalPrecision.precision * 100).toFixed(1)}%
731 </div>
732 <div class="status-stat-label">
733 Precision · {signalPrecision.judged} verdicts
734 </div>
735 </div>
736 <div class="status-stat">
737 <div class="status-stat-num">{signalPrecision.acted}</div>
738 <div class="status-stat-label">Judged useful</div>
739 </div>
740 <div class="status-stat">
741 <div class="status-stat-num">
742 {signalPrecision.dismissed}
743 </div>
744 <div class="status-stat-label">Judged wrong</div>
745 </div>
746 <div class="status-stat">
747 <div class="status-stat-num">
748 {signalPrecision.judged}/{signalPrecision.totalSignals}
749 </div>
750 <div class="status-stat-label">
751 Findings judged · {SIGNAL_WINDOW_DAYS}d
752 </div>
753 </div>
754 </div>
755 )}
756 <p class="status-hero-method">
757 Computed from developer verdicts on AI review findings, not
758 claimed. Each finding on a pull request carries a one-click
759 verdict — useful or wrong — from the PR author or a reviewer
760 with write access; precision is judged-useful divided by all
761 judged, and always publishes beside its sample size.
762 </p>
763 </div>
764 </section>
765
652766 {/* ─── Service uptime badges ─── */}
653767 <section class="status-section" aria-labelledby="status-comp-h">
654768 <header class="status-section-head">
655769
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts