feat(lfs): Git LFS batch API + basic transfer — content-addressed disk store #5512
5 changed files+1209−0
Modified.env.example+5−0View fileUnifiedSplit
@@ -332,3 +332,8 @@ TRUSTED_PROXY_HOPS=1
332332# SSH_DEBUG=
333333# Force SSRF guard enforcement inside the test suite (normally relaxed).
334334# SSRF_ENFORCE_IN_TEST=
335
336# ── Git LFS (2026-08-22) ──────────────────────────────────────────────────
337# Largest single LFS object accepted (bytes). Default 512 MiB. Objects are
338# stored content-addressed under ${GIT_REPOS_PATH}/.lfs/ on the repos volume.
339LFS_MAX_OBJECT_BYTES=
Addedsrc/__tests__/lfs.test.ts+544−0View fileUnifiedSplit
@@ -0,0 +1,544 @@
1/**
2 * Git LFS — storage lib (src/lib/lfs-store.ts) + routes (src/routes/lfs.ts).
3 *
4 * Storage tests run against a mkdtemp root and exercise the invariants that
5 * matter: an object is only ever visible under its oid when the bytes hash
6 * to that oid (tempfile → verify → rename), failed uploads keep nothing on
7 * disk, concurrent duplicate uploads both succeed, and the owner/repo path
8 * segments are lowercased to match the platform's case-insensitive repo
9 * resolution.
10 *
11 * Route tests run the router without a database: the auth gate walks
12 * users/repositories through drizzle in production, so these tests install
13 * the __setLfsAccessGateForTests seam and point GIT_REPOS_PATH at a temp
14 * tree containing a bare-repo HEAD (repoExists is a disk check). Access
15 * SEMANTICS are the gate's own concern and mirror gitAccessGate in
16 * routes/git.ts — what's asserted here is that every endpoint consults the
17 * gate with the right required level and honors its denial.
18 */
19
20import { describe, it, expect, beforeAll, afterAll } from "bun:test";
21import { join } from "path";
22import { tmpdir } from "os";
23import { mkdtemp, mkdir, rm, readdir } from "fs/promises";
24import { Hono } from "hono";
25
26import {
27 DEFAULT_LFS_MAX_OBJECT_BYTES,
28 isValidLfsOid,
29 lfsMaxObjectBytes,
30 lfsObjectPath,
31 statLfsObject,
32 storeLfsObject,
33} from "../lib/lfs-store";
34import lfsRoutes, { __setLfsAccessGateForTests } from "../routes/lfs";
35import { __resetRepoCaseIndex } from "../git/repository";
36
37function sha256hex(data: Uint8Array | string): string {
38 const h = new Bun.CryptoHasher("sha256");
39 h.update(data);
40 return h.digest("hex");
41}
42
43const bytes = (s: string) => new TextEncoder().encode(s);
44
45/** Every regular file under dir, recursively. Missing dir counts as empty. */
46async function listFiles(dir: string): Promise<string[]> {
47 const out: string[] = [];
48 let entries;
49 try {
50 entries = await readdir(dir, { withFileTypes: true });
51 } catch {
52 return out;
53 }
54 for (const e of entries) {
55 const p = join(dir, e.name);
56 if (e.isDirectory()) out.push(...(await listFiles(p)));
57 else out.push(p);
58 }
59 return out;
60}
61
62// ---------------------------------------------------------------------------
63// Storage lib
64// ---------------------------------------------------------------------------
65
66describe("lfs-store — oid and path validation", () => {
67 it("accepts only lowercase 64-hex oids", () => {
68 expect(isValidLfsOid(sha256hex("x"))).toBe(true);
69 expect(isValidLfsOid(sha256hex("x").toUpperCase())).toBe(false);
70 expect(isValidLfsOid("abc")).toBe(false);
71 expect(isValidLfsOid("")).toBe(false);
72 expect(isValidLfsOid("g".repeat(64))).toBe(false);
73 });
74
75 it("lowercases owner/repo and fans out on oid prefix", () => {
76 const oid = sha256hex("x");
77 const p = lfsObjectPath("/root", "Alice", "Widgets", oid);
78 const norm = p.replace(/\\/g, "/");
79 expect(norm).toBe(
80 `/root/.lfs/alice/widgets/${oid.slice(0, 2)}/${oid.slice(2, 4)}/${oid}`
81 );
82 });
83
84 it("refuses traversal-shaped segments and invalid oids", () => {
85 const oid = sha256hex("x");
86 expect(() => lfsObjectPath("/root", "..", "repo", oid)).toThrow();
87 expect(() => lfsObjectPath("/root", "a/b", "repo", oid)).toThrow();
88 expect(() => lfsObjectPath("/root", "owner", "..", oid)).toThrow();
89 expect(() => lfsObjectPath("/root", "owner", "repo", "nothex")).toThrow();
90 });
91
92 it("max size: default applies, env overrides, garbage falls back", () => {
93 const prev = process.env.LFS_MAX_OBJECT_BYTES;
94 try {
95 delete process.env.LFS_MAX_OBJECT_BYTES;
96 expect(lfsMaxObjectBytes()).toBe(DEFAULT_LFS_MAX_OBJECT_BYTES);
97 process.env.LFS_MAX_OBJECT_BYTES = "1024";
98 expect(lfsMaxObjectBytes()).toBe(1024);
99 process.env.LFS_MAX_OBJECT_BYTES = "banana";
100 expect(lfsMaxObjectBytes()).toBe(DEFAULT_LFS_MAX_OBJECT_BYTES);
101 process.env.LFS_MAX_OBJECT_BYTES = "-5";
102 expect(lfsMaxObjectBytes()).toBe(DEFAULT_LFS_MAX_OBJECT_BYTES);
103 } finally {
104 if (prev === undefined) delete process.env.LFS_MAX_OBJECT_BYTES;
105 else process.env.LFS_MAX_OBJECT_BYTES = prev;
106 }
107 });
108});
109
110describe("lfs-store — storeLfsObject", () => {
111 let root = "";
112
113 beforeAll(async () => {
114 root = await mkdtemp(join(tmpdir(), "lfs-store-"));
115 });
116
117 afterAll(async () => {
118 await rm(root, { recursive: true, force: true });
119 });
120
121 it("roundtrips: store → stat → read back the same bytes", async () => {
122 const content = bytes("hello large file world");
123 const oid = sha256hex(content);
124
125 expect(await statLfsObject(root, "alice", "widgets", oid)).toBeNull();
126
127 const res = await storeLfsObject(root, "alice", "widgets", oid, content, {
128 declaredSize: content.byteLength,
129 });
130 expect(res.ok).toBe(true);
131 if (res.ok) expect(res.size).toBe(content.byteLength);
132
133 expect(await statLfsObject(root, "alice", "widgets", oid)).toBe(
134 content.byteLength
135 );
136 const stored = await Bun.file(
137 lfsObjectPath(root, "alice", "widgets", oid)
138 ).bytes();
139 expect(stored).toEqual(content);
140 });
141
142 it("stores under the lowercased owner/repo regardless of caller casing", async () => {
143 const content = bytes("case-insensitive resolution");
144 const oid = sha256hex(content);
145 const res = await storeLfsObject(root, "Alice", "Widgets", oid, content);
146 expect(res.ok).toBe(true);
147 // Visible under lowercase — the path both casings resolve to.
148 expect(await statLfsObject(root, "alice", "widgets", oid)).toBe(
149 content.byteLength
150 );
151 });
152
153 it("accepts a ReadableStream body", async () => {
154 const content = bytes("streamed in two chunks");
155 const oid = sha256hex(content);
156 const stream = new ReadableStream<Uint8Array>({
157 start(controller) {
158 controller.enqueue(content.slice(0, 7));
159 controller.enqueue(content.slice(7));
160 controller.close();
161 },
162 });
163 const res = await storeLfsObject(root, "alice", "widgets", oid, stream);
164 expect(res.ok).toBe(true);
165 expect(await statLfsObject(root, "alice", "widgets", oid)).toBe(
166 content.byteLength
167 );
168 });
169
170 it("rejects a hash mismatch and keeps nothing — not even a temp file", async () => {
171 const wrongOid = sha256hex("the bytes I claimed");
172 const res = await storeLfsObject(
173 root,
174 "bob",
175 "assets",
176 wrongOid,
177 bytes("the bytes I sent")
178 );
179 expect(res).toEqual({ ok: false, reason: "hash-mismatch", size: 16 });
180 expect(await statLfsObject(root, "bob", "assets", wrongOid)).toBeNull();
181 expect(await listFiles(join(root, ".lfs", "bob"))).toEqual([]);
182 });
183
184 it("rejects a declared-size mismatch", async () => {
185 const content = bytes("size lies");
186 const oid = sha256hex(content);
187 const res = await storeLfsObject(root, "bob", "assets", oid, content, {
188 declaredSize: content.byteLength + 1,
189 });
190 expect(res.ok).toBe(false);
191 if (!res.ok) expect(res.reason).toBe("size-mismatch");
192 expect(await statLfsObject(root, "bob", "assets", oid)).toBeNull();
193 });
194
195 it("rejects objects over maxBytes and keeps nothing", async () => {
196 const content = bytes("x".repeat(100));
197 const oid = sha256hex(content);
198 const res = await storeLfsObject(root, "bob", "assets", oid, content, {
199 maxBytes: 10,
200 });
201 expect(res.ok).toBe(false);
202 if (!res.ok) expect(res.reason).toBe("too-large");
203 expect(await statLfsObject(root, "bob", "assets", oid)).toBeNull();
204 expect(await listFiles(join(root, ".lfs", "bob"))).toEqual([]);
205 });
206
207 it("concurrent duplicate uploads of the same oid both succeed", async () => {
208 const content = bytes("duplicated concurrently".repeat(1000));
209 const oid = sha256hex(content);
210 const [a, b] = await Promise.all([
211 storeLfsObject(root, "carol", "media", oid, content),
212 storeLfsObject(root, "carol", "media", oid, content),
213 ]);
214 expect(a.ok).toBe(true);
215 expect(b.ok).toBe(true);
216 expect(await statLfsObject(root, "carol", "media", oid)).toBe(
217 content.byteLength
218 );
219 // Exactly the object file — the losing temp file was cleaned up.
220 const files = await listFiles(join(root, ".lfs", "carol"));
221 expect(files.length).toBe(1);
222 expect(files[0]!.endsWith(oid)).toBe(true);
223 });
224
225 it("re-uploading an existing object succeeds (idempotent)", async () => {
226 const content = bytes("stored twice sequentially");
227 const oid = sha256hex(content);
228 const first = await storeLfsObject(root, "carol", "media", oid, content);
229 const second = await storeLfsObject(root, "carol", "media", oid, content);
230 expect(first.ok).toBe(true);
231 expect(second.ok).toBe(true);
232 expect(await statLfsObject(root, "carol", "media", oid)).toBe(
233 content.byteLength
234 );
235 });
236});
237
238// ---------------------------------------------------------------------------
239// Routes
240// ---------------------------------------------------------------------------
241
242describe("LFS routes", () => {
243 let reposRoot = "";
244 let prevReposPath: string | undefined;
245 const app = new Hono();
246 app.route("/", lfsRoutes);
247
248 // Gate behavior is switched per-test; the default grants everything, which
249 // is the anonymous-public-repo case.
250 let denyWrite = false;
251 let denyAsNotFound = false;
252 const gateCalls: Array<{ required: string; mode: string }> = [];
253
254 const OWNER = "alice";
255 const REPO = "widgets";
256 const BATCH = `/${OWNER}/${REPO}.git/info/lfs/objects/batch`;
257 const objUrl = (oid: string) => `/${OWNER}/${REPO}.git/info/lfs/objects/${oid}`;
258
259 const CONTENT = bytes("route-level roundtrip payload");
260 const OID = sha256hex(CONTENT);
261 const MISSING_OID = sha256hex("never uploaded");
262
263 function batchReq(body: unknown, headers: Record<string, string> = {}) {
264 return app.request(BATCH, {
265 method: "POST",
266 headers: {
267 "content-type": "application/vnd.git-lfs+json",
268 accept: "application/vnd.git-lfs+json",
269 ...headers,
270 },
271 body: JSON.stringify(body),
272 });
273 }
274
275 beforeAll(async () => {
276 reposRoot = await mkdtemp(join(tmpdir(), "lfs-routes-"));
277 prevReposPath = process.env.GIT_REPOS_PATH;
278 process.env.GIT_REPOS_PATH = reposRoot;
279 // repoExists is a disk check on <root>/<owner>/<repo>.git/HEAD.
280 await mkdir(join(reposRoot, OWNER, `${REPO}.git`), { recursive: true });
281 await Bun.write(
282 join(reposRoot, OWNER, `${REPO}.git`, "HEAD"),
283 "ref: refs/heads/main\n"
284 );
285 // The case-resolution memo is keyed on disk state from before this root existed.
286 __resetRepoCaseIndex();
287
288 __setLfsAccessGateForTests(async (_c, _owner, _repo, required, mode) => {
289 gateCalls.push({ required, mode });
290 if (denyAsNotFound) {
291 return new Response(JSON.stringify({ message: "Repository not found" }), {
292 status: 404,
293 headers: { "Content-Type": "application/vnd.git-lfs+json" },
294 });
295 }
296 if (denyWrite && required === "write") {
297 return new Response(
298 JSON.stringify({ message: "You do not have write access to this repository." }),
299 { status: 403, headers: { "Content-Type": "application/vnd.git-lfs+json" } }
300 );
301 }
302 return null;
303 });
304 });
305
306 afterAll(async () => {
307 __setLfsAccessGateForTests(null);
308 if (prevReposPath === undefined) delete process.env.GIT_REPOS_PATH;
309 else process.env.GIT_REPOS_PATH = prevReposPath;
310 __resetRepoCaseIndex();
311 await rm(reposRoot, { recursive: true, force: true });
312 });
313
314 it("batch: unknown repo → 404 before any object work", async () => {
315 const res = await app.request(`/ghost/none.git/info/lfs/objects/batch`, {
316 method: "POST",
317 body: JSON.stringify({ operation: "download", objects: [] }),
318 });
319 expect(res.status).toBe(404);
320 });
321
322 it("batch: invalid operation → 422", async () => {
323 const res = await batchReq({ operation: "delete", objects: [] });
324 expect(res.status).toBe(422);
325 });
326
327 it("batch: transfers list without basic → 422", async () => {
328 const res = await batchReq({
329 operation: "download",
330 transfers: ["multipart"],
331 objects: [],
332 });
333 expect(res.status).toBe(422);
334 });
335
336 it("batch download of a missing object → per-object 404 error entry", async () => {
337 const res = await batchReq({
338 operation: "download",
339 objects: [{ oid: MISSING_OID, size: 14 }],
340 });
341 expect(res.status).toBe(200);
342 expect(res.headers.get("content-type")).toBe("application/vnd.git-lfs+json");
343 const body = await res.json();
344 expect(body.transfer).toBe("basic");
345 expect(body.objects.length).toBe(1);
346 expect(body.objects[0].oid).toBe(MISSING_OID);
347 expect(body.objects[0].error).toEqual({
348 code: 404,
349 message: "Object does not exist",
350 });
351 expect(body.objects[0].actions).toBeUndefined();
352 });
353
354 it("batch upload of a missing object → upload + verify actions with passthrough auth", async () => {
355 const res = await batchReq(
356 { operation: "upload", objects: [{ oid: MISSING_OID, size: 14 }] },
357 { authorization: "Basic eDp0b2tlbg==" }
358 );
359 expect(res.status).toBe(200);
360 const body = await res.json();
361 const obj = body.objects[0];
362 expect(obj.error).toBeUndefined();
363 expect(obj.actions.upload.href).toContain(
364 `/${OWNER}/${REPO}.git/info/lfs/objects/${MISSING_OID}`
365 );
366 expect(obj.actions.upload.header.Authorization).toBe("Basic eDp0b2tlbg==");
367 expect(obj.actions.verify.href.endsWith(`${MISSING_OID}/verify`)).toBe(true);
368 });
369
370 it("batch upload rejects invalid oids per-object without failing the batch", async () => {
371 const res = await batchReq({
372 operation: "upload",
373 objects: [
374 { oid: "NOT-AN-OID", size: 5 },
375 { oid: MISSING_OID, size: 5 },
376 ],
377 });
378 expect(res.status).toBe(200);
379 const body = await res.json();
380 expect(body.objects[0].error.code).toBe(422);
381 expect(body.objects[1].actions.upload).toBeDefined();
382 });
383
384 it("batch upload requires write access; download only read", async () => {
385 denyWrite = true;
386 try {
387 const up = await batchReq({
388 operation: "upload",
389 objects: [{ oid: MISSING_OID, size: 14 }],
390 });
391 expect(up.status).toBe(403);
392 const down = await batchReq({
393 operation: "download",
394 objects: [{ oid: MISSING_OID, size: 14 }],
395 });
396 expect(down.status).toBe(200);
397 } finally {
398 denyWrite = false;
399 }
400 });
401
402 it("PUT stores an object; GET streams it back; verify confirms it", async () => {
403 const put = await app.request(objUrl(OID), {
404 method: "PUT",
405 headers: { "content-length": String(CONTENT.byteLength) },
406 body: CONTENT,
407 });
408 expect(put.status).toBe(200);
409
410 const get = await app.request(objUrl(OID));
411 expect(get.status).toBe(200);
412 expect(get.headers.get("content-type")).toBe("application/octet-stream");
413 expect(get.headers.get("content-length")).toBe(String(CONTENT.byteLength));
414 expect(new Uint8Array(await get.arrayBuffer())).toEqual(CONTENT);
415
416 const verify = await app.request(`${objUrl(OID)}/verify`, {
417 method: "POST",
418 headers: { "content-type": "application/vnd.git-lfs+json" },
419 body: JSON.stringify({ oid: OID, size: CONTENT.byteLength }),
420 });
421 expect(verify.status).toBe(200);
422 const vbody = await verify.json();
423 expect(vbody).toEqual({ oid: OID, size: CONTENT.byteLength });
424 });
425
426 it("batch upload of an existing object → no actions (server already has it)", async () => {
427 const res = await batchReq({
428 operation: "upload",
429 objects: [{ oid: OID, size: CONTENT.byteLength }],
430 });
431 const body = await res.json();
432 expect(body.objects[0].actions).toBeUndefined();
433 expect(body.objects[0].error).toBeUndefined();
434 });
435
436 it("batch download of an existing object → download action + stored size", async () => {
437 const res = await batchReq({
438 operation: "download",
439 objects: [{ oid: OID, size: CONTENT.byteLength }],
440 });
441 const body = await res.json();
442 expect(body.objects[0].size).toBe(CONTENT.byteLength);
443 expect(body.objects[0].actions.download.href).toContain(objUrl(OID));
444 expect(body.objects[0].error).toBeUndefined();
445 });
446
447 it("PUT rejects a body that does not hash to the oid and keeps nothing", async () => {
448 const put = await app.request(objUrl(MISSING_OID), {
449 method: "PUT",
450 body: bytes("these are not the declared bytes"),
451 });
452 expect(put.status).toBe(422);
453 const get = await app.request(objUrl(MISSING_OID));
454 expect(get.status).toBe(404);
455 });
456
457 it("PUT enforces LFS_MAX_OBJECT_BYTES with 413 (declared and actual)", async () => {
458 const prev = process.env.LFS_MAX_OBJECT_BYTES;
459 process.env.LFS_MAX_OBJECT_BYTES = "8";
460 try {
461 const content = bytes("way past eight bytes");
462 const oid = sha256hex(content);
463 // Declared over the cap: refused before the body is read.
464 const declared = await app.request(objUrl(oid), {
465 method: "PUT",
466 headers: { "content-length": String(content.byteLength) },
467 body: content,
468 });
469 expect(declared.status).toBe(413);
470 // No content-length: caught while streaming.
471 const streamed = await app.request(objUrl(oid), {
472 method: "PUT",
473 body: new ReadableStream<Uint8Array>({
474 start(controller) {
475 controller.enqueue(content);
476 controller.close();
477 },
478 }),
479 // @ts-expect-error — Bun needs duplex for stream bodies; not in the lib type.
480 duplex: "half",
481 });
482 expect(streamed.status).toBe(413);
483 const after = await app.request(objUrl(oid));
484 expect(after.status).toBe(404);
485 } finally {
486 if (prev === undefined) delete process.env.LFS_MAX_OBJECT_BYTES;
487 else process.env.LFS_MAX_OBJECT_BYTES = prev;
488 }
489 });
490
491 it("PUT requires write access", async () => {
492 denyWrite = true;
493 try {
494 const res = await app.request(objUrl(MISSING_OID), {
495 method: "PUT",
496 body: bytes("denied"),
497 });
498 expect(res.status).toBe(403);
499 } finally {
500 denyWrite = false;
501 }
502 });
503
504 it("GET/verify honor a privacy-preserving 404 from the gate", async () => {
505 denyAsNotFound = true;
506 try {
507 expect((await app.request(objUrl(OID))).status).toBe(404);
508 const verify = await app.request(`${objUrl(OID)}/verify`, {
509 method: "POST",
510 body: JSON.stringify({ oid: OID, size: CONTENT.byteLength }),
511 });
512 expect(verify.status).toBe(404);
513 } finally {
514 denyAsNotFound = false;
515 }
516 });
517
518 it("verify: missing object → 404; wrong size → 404", async () => {
519 const missing = await app.request(`${objUrl(MISSING_OID)}/verify`, {
520 method: "POST",
521 body: JSON.stringify({ oid: MISSING_OID, size: 14 }),
522 });
523 expect(missing.status).toBe(404);
524
525 const wrongSize = await app.request(`${objUrl(OID)}/verify`, {
526 method: "POST",
527 body: JSON.stringify({ oid: OID, size: CONTENT.byteLength + 1 }),
528 });
529 expect(wrongSize.status).toBe(404);
530 });
531
532 it("locks endpoints return the spec's empty shapes", async () => {
533 const locks = await app.request(`/${OWNER}/${REPO}.git/info/lfs/locks`);
534 expect(locks.status).toBe(200);
535 expect(await locks.json()).toEqual({ locks: [] });
536
537 const verify = await app.request(
538 `/${OWNER}/${REPO}.git/info/lfs/locks/verify`,
539 { method: "POST", body: JSON.stringify({}) }
540 );
541 expect(verify.status).toBe(200);
542 expect(await verify.json()).toEqual({ ours: [], theirs: [] });
543 });
544});
Modifiedsrc/app.tsx+6−0View fileUnifiedSplit
@@ -10,6 +10,7 @@ import { reportError } from "./lib/observability";
1010import { requestContext } from "./middleware/request-context";
1111import { rateLimit } from "./middleware/rate-limit";
1212import gitRoutes from "./routes/git";
13import lfsRoutes from "./routes/lfs";
1314import apiRoutes from "./routes/api";
1415import apiV2Routes from "./routes/api-v2";
1516import apiDocsRoutes from "./routes/api-docs";
@@ -557,6 +558,11 @@ app.use("/:owner/:repo/*", async (c, next) => {
557558// Git Smart HTTP protocol routes (must be before web routes)
558559app.route("/", gitRoutes);
559560
561// Git LFS batch API + basic transfer — /:owner/:repo.git/info/lfs/*.
562// Distinct paths from gitRoutes, mounted beside it because both serve the
563// same .git URL namespace with the same auth semantics.
564app.route("/", lfsRoutes);
565
560566// REST API v1 (legacy)
561567app.route("/", apiRoutes);
562568
Addedsrc/lib/lfs-store.ts+192−0View fileUnifiedSplit
@@ -0,0 +1,192 @@
1/**
2 * Git LFS object store — content-addressed blobs on disk, no database.
3 *
4 * Layout: `${GIT_REPOS_PATH}/.lfs/<owner>/<repo>/<oid[0:2]>/<oid[2:4]>/<oid>`
5 * with owner/repo lowercased. Repo resolution is case-insensitive everywhere
6 * else (see the repoPath comment in src/git/repository.ts); a case-sensitive
7 * path here would split one repo's objects across directories depending on
8 * which casing the client's remote URL happened to use.
9 *
10 * Writes are tempfile → fsync → verify → rename. The verify happens BEFORE
11 * the rename, so a corrupt or truncated upload never becomes visible under
12 * its oid. The rename is atomic-replace (POSIX rename; MOVEFILE_REPLACE on
13 * Windows via libuv), so a reader never observes a half-written object and
14 * two concurrent uploads of the same oid both succeed — the bytes are
15 * identical by definition, the name IS the sha256, so last-rename-wins is
16 * correct. On Windows the replace can fail if the destination is open; that
17 * case is treated as success when the destination already holds the right
18 * byte count, because it means the other upload won.
19 */
20
21import { join, dirname } from "path";
22import { mkdir, open, rename, stat, unlink } from "fs/promises";
23import type { FileHandle } from "fs/promises";
24import { config } from "./config";
25
26/**
27 * Hard cap on a single LFS object. 512 MiB default — the same order as the
28 * push-body cap in routes/git.ts and for the same reason: uploads land on the
29 * one box that runs everything else. Tune with LFS_MAX_OBJECT_BYTES.
30 */
31export const DEFAULT_LFS_MAX_OBJECT_BYTES = 512 * 1024 * 1024;
32
33export function lfsMaxObjectBytes(): number {
34 const v = Number(process.env.LFS_MAX_OBJECT_BYTES);
35 return Number.isFinite(v) && v > 0 ? Math.floor(v) : DEFAULT_LFS_MAX_OBJECT_BYTES;
36}
37
38/** LFS oids are lowercase sha256 hex. Anything else never touches the disk. */
39export function isValidLfsOid(oid: string): boolean {
40 return typeof oid === "string" && /^[0-9a-f]{64}$/.test(oid);
41}
42
43/**
44 * Absolute path of an object. The oid is validated by every caller before
45 * this runs, but validate again — this function is the last line between a
46 * caller-supplied string and a filesystem path.
47 */
48export function lfsObjectPath(
49 root: string,
50 owner: string,
51 repo: string,
52 oid: string
53): string {
54 if (!isValidLfsOid(oid)) {
55 throw new Error("invalid LFS oid");
56 }
57 // Owner/repo arrive as URL path segments. Lowercasing handles casing; the
58 // separator/dot check refuses traversal shapes ("..", "a/b") that would
59 // escape the .lfs tree.
60 const o = owner.toLowerCase();
61 const r = repo.toLowerCase();
62 if (!/^[a-z0-9][a-z0-9._-]*$/.test(o) || !/^[a-z0-9][a-z0-9._-]*$/.test(r) ||
63 o.includes("..") || r.includes("..")) {
64 throw new Error("invalid LFS repo path segment");
65 }
66 return join(root, ".lfs", o, r, oid.slice(0, 2), oid.slice(2, 4), oid);
67}
68
69/** Convenience: the platform's configured repo root. Read lazily (env-driven). */
70export function lfsRoot(): string {
71 return config.gitReposPath;
72}
73
74/** Size in bytes if the object exists, null otherwise. */
75export async function statLfsObject(
76 root: string,
77 owner: string,
78 repo: string,
79 oid: string
80): Promise<number | null> {
81 try {
82 const s = await stat(lfsObjectPath(root, owner, repo, oid));
83 return s.isFile() ? s.size : null;
84 } catch {
85 return null;
86 }
87}
88
89export type LfsStoreResult =
90 | { ok: true; size: number }
91 | { ok: false; reason: "hash-mismatch" | "size-mismatch" | "too-large"; size: number };
92
93/**
94 * Stream an upload into the store.
95 *
96 * The received bytes are hashed incrementally and compared against `oid` —
97 * the client names the object, the server proves the name. A mismatch (or a
98 * declared-size mismatch, or blowing past `maxBytes`) discards the temp file
99 * and keeps nothing: a wrong object under a right name would poison every
100 * future download of that oid, silently, for every clone.
101 */
102export async function storeLfsObject(
103 root: string,
104 owner: string,
105 repo: string,
106 oid: string,
107 body: ReadableStream<Uint8Array> | Uint8Array | null,
108 opts?: { declaredSize?: number; maxBytes?: number }
109): Promise<LfsStoreResult> {
110 const finalPath = lfsObjectPath(root, owner, repo, oid);
111 const dir = dirname(finalPath);
112 await mkdir(dir, { recursive: true });
113
114 const maxBytes = opts?.maxBytes ?? lfsMaxObjectBytes();
115 // Temp file in the destination directory so the rename below can never be
116 // a cross-device copy (which is not atomic). Unique suffix so concurrent
117 // uploads of the same oid don't fight over the temp name.
118 const tmpPath = join(
119 dir,
120 `${oid}.tmp.${process.pid}.${crypto.randomUUID().slice(0, 8)}`
121 );
122
123 let fh: FileHandle | null = null;
124 let total = 0;
125 const hasher = new Bun.CryptoHasher("sha256");
126
127 const discard = async () => {
128 try {
129 await fh?.close();
130 } catch {}
131 fh = null;
132 try {
133 await unlink(tmpPath);
134 } catch {}
135 };
136
137 try {
138 fh = await open(tmpPath, "w");
139 const source: AsyncIterable<Uint8Array> | Iterable<Uint8Array> =
140 body instanceof Uint8Array ? [body] : body ?? [];
141 for await (const chunk of source) {
142 total += chunk.byteLength;
143 if (total > maxBytes) {
144 // Breaking out of for-await cancels the underlying stream, so the
145 // client's remaining bytes are not drained into a doomed upload.
146 await discard();
147 return { ok: false, reason: "too-large", size: total };
148 }
149 hasher.update(chunk);
150 await fh.write(chunk);
151 }
152
153 if (
154 typeof opts?.declaredSize === "number" &&
155 Number.isFinite(opts.declaredSize) &&
156 total !== opts.declaredSize
157 ) {
158 await discard();
159 return { ok: false, reason: "size-mismatch", size: total };
160 }
161
162 const digest = hasher.digest("hex");
163 if (digest !== oid) {
164 await discard();
165 return { ok: false, reason: "hash-mismatch", size: total };
166 }
167
168 await fh.sync();
169 await fh.close();
170 fh = null;
171
172 try {
173 await rename(tmpPath, finalPath);
174 } catch (err) {
175 // Windows: replace fails when the destination is open by a concurrent
176 // duplicate upload or an in-flight download. Content-addressing makes
177 // that a success — the object under this oid holds these exact bytes.
178 const existing = await statLfsObject(root, owner, repo, oid);
179 if (existing !== total) {
180 await discard();
181 throw err;
182 }
183 try {
184 await unlink(tmpPath);
185 } catch {}
186 }
187 return { ok: true, size: total };
188 } catch (err) {
189 await discard();
190 throw err;
191 }
192}
Addedsrc/routes/lfs.ts+462−0View fileUnifiedSplit
@@ -0,0 +1,462 @@
1/**
2 * Git LFS server — "basic" transfer adapter.
3 *
4 * Mounted at /:owner/:repo.git/info/lfs/ (the default path git-lfs derives
5 * from the remote URL, so no `.lfsconfig` is needed in cloned repos):
6 *
7 * POST /objects/batch — the batch API (download/upload negotiation)
8 * PUT /objects/:oid — upload one object (basic transfer)
9 * GET /objects/:oid — download one object
10 * POST /objects/:oid/verify — post-upload existence + size check
11 * GET /locks, POST /locks/verify — empty spec shapes only; file locking
12 * is not implemented, and `git lfs push`
13 * treats these responses as "no locks held".
14 *
15 * LFS is available whenever the server runs — storage is content-addressed
16 * files on disk (src/lib/lfs-store.ts), no database table, no capability
17 * gate. Objects live under `${GIT_REPOS_PATH}/.lfs/`, beside the bare repos
18 * they belong to, so the existing repo-volume backup covers them.
19 *
20 * Auth deliberately mirrors the Smart HTTP layer in routes/git.ts: the
21 * git-lfs client authenticates with the same Basic credentials as the git
22 * remote (resolvePusher handles PAT-in-basic), download needs read access,
23 * upload needs write access, anonymous read of a public repo works exactly
24 * like anonymous clone does. Any drift between the two gates surfaces as
25 * "clone works but lfs fetch 404s", so the gate here must stay behaviorally
26 * identical to gitAccessGate in routes/git.ts (module-private there — this
27 * file cannot import it without widening git.ts's surface).
28 */
29
30import { Hono } from "hono";
31import { and, eq, sql } from "drizzle-orm";
32import { db } from "../db";
33import { repositories, users } from "../db/schema";
34import { repoExists } from "../git/repository";
35import { resolvePusher } from "../lib/git-push-auth";
36import {
37 resolveRepoAccess,
38 satisfiesAccess,
39 type RepoAccessLevel,
40} from "../middleware/repo-access";
41import {
42 isValidLfsOid,
43 lfsMaxObjectBytes,
44 lfsObjectPath,
45 lfsRoot,
46 statLfsObject,
47 storeLfsObject,
48} from "../lib/lfs-store";
49
50const LFS_JSON = "application/vnd.git-lfs+json";
51
52const lfs = new Hono();
53
54/** Same param shape as gitParams in routes/git.ts — strip the ".git". */
55function lfsParams(c: any): { owner: string; repo: string } {
56 const params = c.req.param();
57 const owner: string = params.owner;
58 const raw: string = params["repo.git"] ?? params.repo ?? "";
59 const repo = raw.replace(/\.git$/, "");
60 return { owner, repo };
61}
62
63function lfsJson(
64 c: any,
65 status: number,
66 body: unknown,
67 headers?: Record<string, string>
68): Response {
69 return c.body(JSON.stringify(body), status, {
70 "Content-Type": LFS_JSON,
71 ...headers,
72 });
73}
74
75/** Case-insensitive repo row lookup — same resolution as loadRepoRow in git.ts. */
76async function loadRepoRow(
77 ownerName: string,
78 repoName: string
79): Promise<{ id: string; isPrivate: boolean } | null> {
80 try {
81 const [ownerRow] = await db
82 .select({ id: users.id })
83 .from(users)
84 .where(sql`lower(${users.username}) = lower(${ownerName})`)
85 .limit(1);
86 if (!ownerRow) return null;
87 const [repoRow] = await db
88 .select({ id: repositories.id, isPrivate: repositories.isPrivate })
89 .from(repositories)
90 .where(
91 and(
92 eq(repositories.ownerId, ownerRow.id),
93 sql`lower(${repositories.name}) = lower(${repoName})`
94 )
95 )
96 .limit(1);
97 return repoRow || null;
98 } catch {
99 return null;
100 }
101}
102
103type LfsGate = (
104 c: any,
105 owner: string,
106 repo: string,
107 required: RepoAccessLevel,
108 onInsufficientWithAuth: "notfound" | "forbidden"
109) => Promise<Response | null>;
110
111/**
112 * Test seam — the real gate walks users/repositories/collaborators through
113 * drizzle, and the LFS route tests must run without a database (the storage
114 * layer under test has none). Never set outside tests.
115 */
116let gateOverride: LfsGate | null = null;
117export function __setLfsAccessGateForTests(fn: LfsGate | null): void {
118 gateOverride = fn;
119}
120
121/**
122 * Authorization gate — returns a Response to short-circuit with, or null to
123 * proceed. Semantics in lockstep with gitAccessGate (routes/git.ts):
124 *
125 * - Public repos grant "read" to anonymous callers.
126 * - Insufficient with NO auth → 401 + WWW-Authenticate so the client prompts.
127 * - Insufficient WITH auth → 404 on read gates (privacy-preserving: don't
128 * confirm a private repo exists to a non-collaborator) or 403 on write
129 * gates (existence already implied by the earlier discovery 404s).
130 *
131 * Error bodies are LFS-media-type JSON rather than git.ts's text/plain —
132 * git-lfs surfaces `message` to the user, plain text it does not.
133 */
134async function resolveRepoAccessOrDeny(
135 c: any,
136 owner: string,
137 repo: string,
138 required: RepoAccessLevel,
139 onInsufficientWithAuth: "notfound" | "forbidden"
140): Promise<Response | null> {
141 if (gateOverride) {
142 return gateOverride(c, owner, repo, required, onInsufficientWithAuth);
143 }
144
145 const authHeader = c.req.header("authorization");
146 const repoRow = await loadRepoRow(owner, repo);
147 if (!repoRow) {
148 return lfsJson(c, 404, { message: "Repository not found" });
149 }
150
151 let userId: string | null = null;
152 try {
153 const pusher = await resolvePusher(authHeader);
154 userId = pusher?.userId || null;
155 } catch {
156 userId = null;
157 }
158
159 const access = await resolveRepoAccess({
160 repoId: repoRow.id,
161 userId,
162 isPublic: !repoRow.isPrivate,
163 });
164 if (satisfiesAccess(access, required)) return null;
165
166 if (!authHeader) {
167 return lfsJson(
168 c,
169 401,
170 { message: "Authentication required" },
171 { "WWW-Authenticate": 'Basic realm="Gluecron"' }
172 );
173 }
174 if (onInsufficientWithAuth === "forbidden") {
175 return lfsJson(c, 403, {
176 message: "You do not have write access to this repository.",
177 });
178 }
179 return lfsJson(c, 404, { message: "Repository not found" });
180}
181
182/**
183 * Absolute base for action hrefs. Behind the proxy the request URL is
184 * http://container-ip; the client must be handed the public origin or its
185 * follow-up PUT/GET goes nowhere. Forwarded headers win, request URL is the
186 * dev fallback.
187 */
188function requestBase(c: any): string {
189 const url = new URL(c.req.url);
190 const proto =
191 c.req.header("x-forwarded-proto") || url.protocol.replace(/:$/, "");
192 const host =
193 c.req.header("x-forwarded-host") || c.req.header("host") || url.host;
194 return `${proto}://${host}`;
195}
196
197// POST /:owner/:repo.git/info/lfs/objects/batch
198lfs.post("/:owner/:repo.git/info/lfs/objects/batch", async (c) => {
199 const { owner, repo } = lfsParams(c);
200 if (!(await repoExists(owner, repo))) {
201 return lfsJson(c, 404, { message: "Repository not found" });
202 }
203
204 let body: any;
205 try {
206 body = await c.req.json();
207 } catch {
208 return lfsJson(c, 400, { message: "Invalid JSON body" });
209 }
210
211 const operation = body?.operation;
212 if (operation !== "download" && operation !== "upload") {
213 return lfsJson(c, 422, {
214 message: 'operation must be "download" or "upload"',
215 });
216 }
217
218 // Only the basic adapter exists. An omitted transfers list means basic per
219 // spec; a list that excludes basic is a client we cannot serve.
220 if (
221 Array.isArray(body.transfers) &&
222 body.transfers.length > 0 &&
223 !body.transfers.includes("basic")
224 ) {
225 return lfsJson(c, 422, {
226 message: 'Only the "basic" transfer adapter is supported',
227 });
228 }
229
230 const denied = await resolveRepoAccessOrDeny(
231 c,
232 owner,
233 repo,
234 operation === "upload" ? "write" : "read",
235 operation === "upload" ? "forbidden" : "notfound"
236 );
237 if (denied) return denied;
238
239 const root = lfsRoot();
240 const maxBytes = lfsMaxObjectBytes();
241 const base = requestBase(c);
242 const authHeader = c.req.header("authorization");
243 // Passing the caller's own credentials back as the action header is what
244 // lets git-lfs reuse them on the follow-up PUT/GET without a re-prompt.
245 const actionHeader = authHeader ? { Authorization: authHeader } : undefined;
246 const requested: any[] = Array.isArray(body.objects) ? body.objects : [];
247
248 const objects: any[] = [];
249 for (const req of requested) {
250 const oid = typeof req?.oid === "string" ? req.oid : "";
251 const size = Number(req?.size);
252 if (!isValidLfsOid(oid) || !Number.isInteger(size) || size < 0) {
253 objects.push({
254 oid,
255 size: Number.isInteger(size) && size >= 0 ? size : 0,
256 error: { code: 422, message: "Invalid oid or size" },
257 });
258 continue;
259 }
260
261 const href = `${base}/${owner}/${repo}.git/info/lfs/objects/${oid}`;
262 const existingSize = await statLfsObject(root, owner, repo, oid);
263
264 if (operation === "download") {
265 if (existingSize === null) {
266 objects.push({
267 oid,
268 size,
269 error: { code: 404, message: "Object does not exist" },
270 });
271 } else {
272 objects.push({
273 oid,
274 size: existingSize,
275 authenticated: true,
276 actions: {
277 download: { href, ...(actionHeader ? { header: actionHeader } : {}) },
278 },
279 });
280 }
281 continue;
282 }
283
284 // upload
285 if (existingSize !== null) {
286 // Already stored — no actions means "server has it, skip the transfer".
287 objects.push({ oid, size });
288 continue;
289 }
290 if (size > maxBytes) {
291 objects.push({
292 oid,
293 size,
294 error: {
295 code: 422,
296 message: `Object exceeds maximum size (${maxBytes} bytes)`,
297 },
298 });
299 continue;
300 }
301 objects.push({
302 oid,
303 size,
304 authenticated: true,
305 actions: {
306 upload: { href, ...(actionHeader ? { header: actionHeader } : {}) },
307 verify: {
308 href: `${href}/verify`,
309 ...(actionHeader ? { header: actionHeader } : {}),
310 },
311 },
312 });
313 }
314
315 return lfsJson(c, 200, { transfer: "basic", objects, hash_algo: "sha256" });
316});
317
318// PUT /:owner/:repo.git/info/lfs/objects/:oid — basic-transfer upload
319lfs.put("/:owner/:repo.git/info/lfs/objects/:oid", async (c) => {
320 const { owner, repo } = lfsParams(c);
321 if (!(await repoExists(owner, repo))) {
322 return lfsJson(c, 404, { message: "Repository not found" });
323 }
324 const denied = await resolveRepoAccessOrDeny(c, owner, repo, "write", "forbidden");
325 if (denied) return denied;
326
327 const oid = c.req.param("oid");
328 if (!isValidLfsOid(oid)) {
329 return lfsJson(c, 422, { message: "Invalid oid" });
330 }
331
332 const maxBytes = lfsMaxObjectBytes();
333 // Refuse on the declared length BEFORE reading the body, same posture as
334 // the push-body cap in git.ts — one giant upload must not become an
335 // out-of-disk (or a long doomed stream) for the whole platform.
336 const declaredLen = Number(c.req.header("content-length"));
337 if (Number.isFinite(declaredLen) && declaredLen > maxBytes) {
338 return lfsJson(c, 413, {
339 message: `Object too large (limit ${maxBytes} bytes)`,
340 });
341 }
342
343 let result;
344 try {
345 result = await storeLfsObject(lfsRoot(), owner, repo, oid, c.req.raw.body, {
346 declaredSize: Number.isFinite(declaredLen) ? declaredLen : undefined,
347 maxBytes,
348 });
349 } catch (err) {
350 console.error(`[lfs] store failed for ${owner}/${repo} ${oid}:`, err);
351 return lfsJson(c, 500, { message: "Failed to store object" });
352 }
353
354 if (!result.ok) {
355 if (result.reason === "too-large") {
356 return lfsJson(c, 413, {
357 message: `Object too large (limit ${maxBytes} bytes)`,
358 });
359 }
360 return lfsJson(c, 422, {
361 message:
362 result.reason === "hash-mismatch"
363 ? "Received data does not match oid"
364 : "Received size does not match Content-Length",
365 });
366 }
367 return c.body(null, 200);
368});
369
370// GET /:owner/:repo.git/info/lfs/objects/:oid — basic-transfer download
371lfs.get("/:owner/:repo.git/info/lfs/objects/:oid", async (c) => {
372 const { owner, repo } = lfsParams(c);
373 if (!(await repoExists(owner, repo))) {
374 return lfsJson(c, 404, { message: "Repository not found" });
375 }
376 const denied = await resolveRepoAccessOrDeny(c, owner, repo, "read", "notfound");
377 if (denied) return denied;
378
379 const oid = c.req.param("oid");
380 if (!isValidLfsOid(oid)) {
381 return lfsJson(c, 404, { message: "Object does not exist" });
382 }
383 const file = Bun.file(lfsObjectPath(lfsRoot(), owner, repo, oid));
384 if (!(await file.exists())) {
385 return lfsJson(c, 404, { message: "Object does not exist" });
386 }
387 return new Response(file, {
388 status: 200,
389 headers: {
390 "Content-Type": "application/octet-stream",
391 "Content-Length": String(file.size),
392 },
393 });
394});
395
396// POST /:owner/:repo.git/info/lfs/objects/:oid/verify — post-upload check
397lfs.post("/:owner/:repo.git/info/lfs/objects/:oid/verify", async (c) => {
398 const { owner, repo } = lfsParams(c);
399 if (!(await repoExists(owner, repo))) {
400 return lfsJson(c, 404, { message: "Repository not found" });
401 }
402 const denied = await resolveRepoAccessOrDeny(c, owner, repo, "read", "notfound");
403 if (denied) return denied;
404
405 const oid = c.req.param("oid");
406 if (!isValidLfsOid(oid)) {
407 return lfsJson(c, 404, { message: "Object does not exist" });
408 }
409
410 let body: any = null;
411 try {
412 body = await c.req.json();
413 } catch {
414 // Spec clients always send {oid, size}; a missing body degrades to an
415 // existence-only check rather than failing a finished upload.
416 }
417
418 const storedSize = await statLfsObject(lfsRoot(), owner, repo, oid);
419 if (storedSize === null) {
420 return lfsJson(c, 404, { message: "Object does not exist" });
421 }
422 if (
423 body &&
424 typeof body.size === "number" &&
425 Number.isFinite(body.size) &&
426 body.size !== storedSize
427 ) {
428 return lfsJson(c, 404, {
429 message: `Object size mismatch (stored ${storedSize}, expected ${body.size})`,
430 });
431 }
432 return lfsJson(c, 200, { oid, size: storedSize });
433});
434
435// GET /:owner/:repo.git/info/lfs/locks — locking is not implemented; the
436// empty shape keeps `git lfs push` from erroring on its pre-push lock probe.
437lfs.get("/:owner/:repo.git/info/lfs/locks", async (c) => {
438 const { owner, repo } = lfsParams(c);
439 if (!(await repoExists(owner, repo))) {
440 return lfsJson(c, 404, { message: "Repository not found" });
441 }
442 const denied = await resolveRepoAccessOrDeny(c, owner, repo, "read", "notfound");
443 if (denied) return denied;
444 return lfsJson(c, 200, { locks: [] });
445});
446
447// POST /:owner/:repo.git/info/lfs/locks/verify — same: always "no locks".
448// Read gate, not write: the probe runs before any object transfer, and a
449// read-only caller learning "no locks exist" discloses nothing.
450lfs.post("/:owner/:repo.git/info/lfs/locks/verify", async (c) => {
451 const { owner, repo } = lfsParams(c);
452 if (!(await repoExists(owner, repo))) {
453 return lfsJson(c, 404, { message: "Repository not found" });
454 }
455 const denied = await resolveRepoAccessOrDeny(c, owner, repo, "read", "notfound");
456 if (denied) return denied;
457 return lfsJson(c, 200, { ours: [], theirs: [] });
458});
459
460export const __test = { lfsParams, requestBase };
461
462export default lfs;
0463
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts