fix(import): repair the bulk import that cloned 38 repos but created zero DB rows #3527
1 changed file+70−35
Modifiedsrc/lib/import-helper.ts+70−35View fileUnifiedSplit
@@ -6,7 +6,8 @@
66 * the single-repo and bulk importers can share one code path.
77 */
88
9import { and, eq } from "drizzle-orm";
9import { and, eq, sql } from "drizzle-orm";
10import { existsSync } from "fs";
1011import { mkdir, rm } from "fs/promises";
1112import { join } from "path";
1213import { db } from "../db";
@@ -171,12 +172,19 @@ export async function importOneRepo(
171172 const safeName = sanitizeRepoName(targetName);
172173
173174 try {
174 // Uniqueness in the caller's namespace (owner+name).
175 // Uniqueness in the caller's namespace (owner+name). Compared
176 // case-insensitively: pre-import repos can be stored with their original
177 // casing (ccantynz/Vapron), while sanitizeRepoName lowercases — an exact
178 // match missed those and cloned a duplicate `vapron.git` beside
179 // `Vapron.git` on the (case-sensitive) prod filesystem.
175180 const [existing] = await db
176181 .select()
177182 .from(repositories)
178183 .where(
179 and(eq(repositories.ownerId, ownerId), eq(repositories.name, safeName))
184 and(
185 eq(repositories.ownerId, ownerId),
186 sql`lower(${repositories.name}) = ${safeName}`
187 )
180188 )
181189 .limit(1);
182190
@@ -203,40 +211,59 @@ export async function importOneRepo(
203211 const destPath = join(config.gitReposPath, ownerUsername, `${safeName}.git`);
204212 await mkdir(join(config.gitReposPath, ownerUsername), { recursive: true });
205213
206 const authedCloneUrl = buildCloneUrl(cloneUrl, token);
207
208 const proc = Bun.spawn(
209 ["git", "clone", "--bare", "--mirror", authedCloneUrl, destPath],
210 {
211 stdout: "pipe",
212 stderr: "pipe",
213 env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
214 // No DB row, but the destination may already hold a full clone: a prior
215 // import that failed AFTER the clone (e.g. the DB insert died) leaves the
216 // history on disk with nothing pointing at it, and re-cloning into it
217 // fails with "destination path already exists". If that orphan has
218 // commits, adopt it instead of failing; if it's empty, clear it and
219 // clone fresh.
220 let commitCount = 0;
221 let adopted = false;
222 if (existsSync(destPath)) {
223 commitCount = await countRepoCommits(destPath);
224 if (commitCount > 0) {
225 adopted = true;
226 } else {
227 await removeTempDir(destPath, "empty leftover from a failed import");
214228 }
215 );
216 const stderr = await new Response(proc.stderr).text();
217 const exitCode = await proc.exited;
218
219 if (exitCode !== 0) {
220 return {
221 status: "failed",
222 name: safeName,
223 notes: `git clone failed: ${scrubSecrets(stderr, token).slice(0, 200)}`,
224 };
225229 }
226230
227 // Gate success on the clone having actually transferred history. `git
228 // clone --mirror` can exit 0 yet leave an empty repo; without this check
229 // that got a DB row + "success" — a phantom empty repo. Clean up the
230 // empty dir (so a retry isn't blocked by "already exists") and fail loud.
231 const commitCount = await countRepoCommits(destPath);
232 if (commitCount === 0) {
233 await removeTempDir(destPath, "import of an empty repository");
234 return {
235 status: "failed",
236 name: safeName,
237 notes:
238 "Clone produced an EMPTY repository — nothing was imported. The source may be empty, or private and the token lacks access. No repo was created; fix access and retry.",
239 };
231 if (!adopted) {
232 const authedCloneUrl = buildCloneUrl(cloneUrl, token);
233
234 const proc = Bun.spawn(
235 ["git", "clone", "--bare", "--mirror", authedCloneUrl, destPath],
236 {
237 stdout: "pipe",
238 stderr: "pipe",
239 env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
240 }
241 );
242 const stderr = await new Response(proc.stderr).text();
243 const exitCode = await proc.exited;
244
245 if (exitCode !== 0) {
246 return {
247 status: "failed",
248 name: safeName,
249 notes: `git clone failed: ${scrubSecrets(stderr, token).slice(0, 200)}`,
250 };
251 }
252
253 // Gate success on the clone having actually transferred history. `git
254 // clone --mirror` can exit 0 yet leave an empty repo; without this check
255 // that got a DB row + "success" — a phantom empty repo. Clean up the
256 // empty dir (so a retry isn't blocked by "already exists") and fail loud.
257 commitCount = await countRepoCommits(destPath);
258 if (commitCount === 0) {
259 await removeTempDir(destPath, "import of an empty repository");
260 return {
261 status: "failed",
262 name: safeName,
263 notes:
264 "Clone produced an EMPTY repository — nothing was imported. The source may be empty, or private and the token lacks access. No repo was created; fix access and retry.",
265 };
266 }
240267 }
241268
242269 // The clone has already run by this point, so a lost race leaves a full
@@ -257,6 +284,12 @@ export async function importOneRepo(
257284 })
258285 .onConflictDoNothing({
259286 target: [repositories.ownerId, repositories.name],
287 // repos_owner_name is PARTIAL (WHERE org_id IS NULL, migration 0004).
288 // ON CONFLICT must repeat the predicate or Postgres rejects the whole
289 // insert ("no unique or exclusion constraint matching the ON CONFLICT
290 // specification") — which is how the first bulk import cloned 38
291 // repos to disk and created zero DB rows.
292 where: sql`org_id is null`,
260293 })
261294 .returning({ id: repositories.id });
262295
@@ -279,7 +312,9 @@ export async function importOneRepo(
279312 return {
280313 status: "success",
281314 name: safeName,
282 notes: `Cloned + indexed (${commitCount} commits)`,
315 notes: adopted
316 ? `Adopted an orphaned clone from a previous failed import (${commitCount} commits)`
317 : `Cloned + indexed (${commitCount} commits)`,
283318 };
284319 } catch (err) {
285320 return {
286321
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts