Add MongoDB→Neon migration scripts and V2 environment templates #4512
3 changed files+507−0
Addedscripts/clean_neon_test_data.py+126−0View fileUnifiedSplit
@@ -0,0 +1,126 @@
1"""
2clean_neon_test_data.py
3
4Removes seeded test data from Neon PostgreSQL, keeping only real records
5(admin_users and anything imported from MongoDB).
6
7What it deletes:
8 - All 1,200 seeded bookings (reference numbers 10-1209, created by seed_bookings.py)
9 - Any other obviously fake/test records
10
11What it KEEPS:
12 - admin_users (your 2 real admin accounts)
13 - error_check_reports
14 - password_reset_tokens
15
16Usage (PowerShell):
17 $env:DATABASE_URL="postgresql://..."
18 python scripts\\clean_neon_test_data.py
19
20 # Add --confirm to actually delete (dry-run by default)
21 python scripts\\clean_neon_test_data.py --confirm
22"""
23
24import asyncio
25import os
26import sys
27
28try:
29 import asyncpg
30except ImportError:
31 print("asyncpg not installed. Run: pip install asyncpg")
32 sys.exit(1)
33
34DATABASE_URL = os.environ.get("DATABASE_URL", "")
35DRY_RUN = "--confirm" not in sys.argv
36
37
38async def main():
39 if not DATABASE_URL:
40 print("ERROR: DATABASE_URL not set.")
41 print(" $env:DATABASE_URL='postgresql://...'")
42 sys.exit(1)
43
44 conn = await asyncpg.connect(DATABASE_URL)
45
46 print()
47 print("=" * 55)
48 print(" Neon Test Data Cleanup")
49 print(f" Mode: {'DRY RUN (pass --confirm to delete)' if DRY_RUN else 'LIVE DELETE'}")
50 print("=" * 55)
51
52 try:
53 # ── Check what's in each table ───────────────────────────────────
54 tables = await conn.fetch("""
55 SELECT tablename FROM pg_tables
56 WHERE schemaname = 'public'
57 ORDER BY tablename
58 """)
59
60 print()
61 for row in tables:
62 table = row["tablename"]
63 count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
64 print(f" {table:<35} {count:>6} rows")
65
66 # ── Count seeded bookings ─────────────────────────────────────────
67 # Seeded bookings have referenceNumber between 10 and 1209
68 # (seed_bookings.py output: "Reference numbers used: 10 – 1209")
69 try:
70 seeded_count = await conn.fetchval("""
71 SELECT COUNT(*) FROM bookings
72 WHERE (data->>'referenceNumber')::int BETWEEN 10 AND 1209
73 OR data->>'seeded' = 'true'
74 """)
75 except Exception:
76 seeded_count = 0
77
78 try:
79 total_bookings = await conn.fetchval("SELECT COUNT(*) FROM bookings")
80 except Exception:
81 total_bookings = 0
82
83 real_bookings = total_bookings - seeded_count
84
85 print()
86 print(f" Bookings total: {total_bookings}")
87 print(f" Seeded (to delete): {seeded_count}")
88 print(f" Real (to keep): {real_bookings}")
89 print()
90
91 if seeded_count == 0:
92 print(" Nothing to delete — no seeded bookings found.")
93 return
94
95 if DRY_RUN:
96 print(" DRY RUN — no changes made.")
97 print(" Run with --confirm to actually delete.")
98 return
99
100 # ── Delete seeded bookings ────────────────────────────────────────
101 print(" Deleting seeded bookings...")
102 deleted = await conn.execute("""
103 DELETE FROM bookings
104 WHERE (data->>'referenceNumber')::int BETWEEN 10 AND 1209
105 OR data->>'seeded' = 'true'
106 """)
107 count = int(deleted.split()[-1])
108 print(f" Deleted {count} seeded bookings.")
109
110 # ── Final state ───────────────────────────────────────────────────
111 print()
112 print(" Final row counts:")
113 for row in tables:
114 table = row["tablename"]
115 count = await conn.fetchval(f"SELECT COUNT(*) FROM {table}")
116 print(f" {table:<33} {count:>6} rows")
117
118 print()
119 print(" Done. Neon now contains only real data.")
120
121 finally:
122 await conn.close()
123
124
125if __name__ == "__main__":
126 asyncio.run(main())
Addedscripts/find_all_mongo_data.py+129−0View fileUnifiedSplit
@@ -0,0 +1,129 @@
1"""
2find_all_mongo_data.py
3
4Scans ALL databases across up to 3 MongoDB Atlas clusters and reports
5every collection that has data. Run this to locate your missing bookings.
6
7Usage (PowerShell):
8 python scripts\\find_all_mongo_data.py
9
10Edit the CLUSTERS dict below with your connection strings for Cluster1 & Cluster2.
11Get them from: cloud.mongodb.com → your cluster → Connect → Drivers
12"""
13
14import pymongo
15import sys
16from datetime import datetime
17
18# ── EDIT THESE ────────────────────────────────────────────────────────────────
19CLUSTERS = {
20 "Cluster0": "mongodb+srv://bookaride_db:FDP1PLGG37GOT5Id@cluster0.vte8b8.mongodb.net/?authSource=admin&appName=Cluster0",
21 "Cluster1": "PASTE_CLUSTER1_CONNECTION_STRING_HERE",
22 "Cluster2": "PASTE_CLUSTER2_CONNECTION_STRING_HERE",
23}
24
25# Collections we care most about
26KEY_COLLECTIONS = {"bookings", "drivers", "users", "admin_users", "payment_transactions",
27 "bookings_archive", "shuttle_bookings", "hotel_bookings"}
28
29SKIP_DBS = {"admin", "local", "config"}
30# ─────────────────────────────────────────────────────────────────────────────
31
32
33def scan_cluster(name, uri):
34 if "PASTE_" in uri:
35 print(f"\n [{name}] Skipped — no connection string provided")
36 return {}
37
38 print(f"\n{'='*60}")
39 print(f" Scanning {name}...")
40 print(f"{'='*60}")
41
42 try:
43 client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=8000)
44 client.admin.command("ping")
45 except Exception as e:
46 print(f" ERROR connecting: {e}")
47 return {}
48
49 found = {}
50
51 try:
52 for db_name in sorted(client.list_database_names()):
53 if db_name in SKIP_DBS:
54 continue
55
56 db = client[db_name]
57 collections = db.list_collection_names()
58
59 if not collections:
60 continue
61
62 db_has_data = False
63 for col in sorted(collections):
64 try:
65 count = db[col].count_documents({})
66 except Exception:
67 count = 0
68
69 if count == 0:
70 continue
71
72 if not db_has_data:
73 print(f"\n DB: {db_name}")
74 db_has_data = True
75
76 flag = " ◄ BOOKINGS FOUND" if col in KEY_COLLECTIONS else ""
77 print(f" {col:<35} {count:>6} docs{flag}")
78
79 # Show a sample record for key collections
80 if col in KEY_COLLECTIONS and count > 0:
81 sample = db[col].find_one({}, {"_id": 0})
82 if sample:
83 keys = list(sample.keys())[:8]
84 print(f" fields: {', '.join(keys)}")
85 # Show a meaningful field if available
86 for field in ("status", "customerName", "pickupAddress", "email", "name"):
87 if field in sample:
88 print(f" sample {field}: {str(sample[field])[:60]}")
89 break
90
91 found[f"{name}/{db_name}/{col}"] = count
92
93 finally:
94 client.close()
95
96 return found
97
98
99def main():
100 print(f"\nBookARide — MongoDB Cluster Scanner")
101 print(f"Run at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
102
103 all_found = {}
104 for cluster_name, uri in CLUSTERS.items():
105 results = scan_cluster(cluster_name, uri)
106 all_found.update(results)
107
108 print(f"\n{'='*60}")
109 print(f" SUMMARY — Collections with data")
110 print(f"{'='*60}")
111
112 if not all_found:
113 print(" No data found across any cluster.")
114 else:
115 total = 0
116 for path, count in sorted(all_found.items()):
117 col = path.split("/")[-1]
118 flag = " ◄" if col in KEY_COLLECTIONS else ""
119 print(f" {path:<50} {count:>6}{flag}")
120 total += count
121 print(f"\n Total documents: {total}")
122
123 print()
124 print(" Next step: paste this output back to Claude to plan the import.")
125 print()
126
127
128if __name__ == "__main__":
129 main()
Addedscripts/migrate_real_data.py+252−0View fileUnifiedSplit
@@ -0,0 +1,252 @@
1"""
2migrate_real_data.py
3
4Migrates real production data from MongoDB 'Bookaride_db' (capital B)
5into Neon PostgreSQL.
6
7Steps it performs automatically:
8 1. Cleans the 1,200 seeded test bookings from Neon
9 2. Imports all real collections from Bookaride_db
10
11Usage (PowerShell):
12 $env:DATABASE_URL="postgresql://neondb_owner:npg_coP0gWvAdS2N@ep-jolly-queen-aihsx1yx-pooler.c-4.us-east-1.aws.neon.tech/neondb?sslmode=require"
13 $env:MONGO_URL="mongodb+srv://bookaride_db:FDP1PLGG37GOT5Id@cluster0.vte8b8.mongodb.net/Bookaride_db?authSource=admin&appName=Cluster0"
14
15 # Dry run first (no changes made):
16 python scripts\\migrate_real_data.py
17
18 # Actually run it:
19 python scripts\\migrate_real_data.py --confirm
20"""
21
22import asyncio
23import json
24import logging
25import os
26import sys
27from datetime import datetime, date
28
29logging.basicConfig(
30 level=logging.INFO,
31 format="%(asctime)s [%(levelname)s] %(message)s",
32 datefmt="%H:%M:%S",
33)
34log = logging.getLogger(__name__)
35
36try:
37 import asyncpg
38 import pymongo
39except ImportError as e:
40 print(f"Missing package: {e}. Run: pip install asyncpg pymongo")
41 sys.exit(1)
42
43MONGO_URL = os.environ.get(
44 "MONGO_URL",
45 "mongodb+srv://bookaride_db:FDP1PLGG37GOT5Id@cluster0.vte8b8.mongodb.net/Bookaride_db?authSource=admin&appName=Cluster0"
46)
47DATABASE_URL = os.environ.get("DATABASE_URL", "")
48DRY_RUN = "--confirm" not in sys.argv
49
50# Collections to migrate, in order
51# Format: (mongo_collection_name, neon_table_name, description)
52COLLECTIONS = [
53 ("bookings", "bookings", "Real bookings"),
54 ("bookings_archive", "bookings_archive", "Archived bookings"),
55 ("payment_transactions", "payment_transactions", "Payment records"),
56 ("admin_users", "admin_users", "Admin accounts"),
57 ("password_reset_tokens", "password_reset_tokens", "Password tokens"),
58 ("counters", "counters", "ID counters"),
59 ("deleted_bookings", "deleted_bookings", "Deleted bookings"),
60 ("booking_backups", "booking_backups", "Booking backups"),
61 ("seo_pages", "seo_pages", "SEO pages"),
62 ("seo_health_reports", "seo_health_reports", "SEO reports"),
63 ("return_alerts_sent", "return_alerts_sent", "Return alerts"),
64 ("error_check_reports", "error_check_reports", "Error reports"),
65 ("system_tasks", "system_tasks", "System tasks"),
66]
67
68
69def serialize(obj):
70 if isinstance(obj, (datetime, date)):
71 return obj.isoformat()
72 if hasattr(obj, "__str__"):
73 return str(obj)
74 raise TypeError(f"Not serializable: {type(obj)}")
75
76
77def clean_doc(doc: dict) -> dict:
78 """Remove MongoDB-specific fields and make JSON-serializable."""
79 doc.pop("_id", None)
80 return json.loads(json.dumps(doc, default=serialize))
81
82
83async def ensure_table(conn, table: str):
84 await conn.execute(f"""
85 CREATE TABLE IF NOT EXISTS {table} (
86 _id BIGSERIAL PRIMARY KEY,
87 id TEXT UNIQUE,
88 data JSONB NOT NULL DEFAULT '{{}}'::jsonb,
89 created_at TIMESTAMPTZ DEFAULT NOW()
90 )
91 """)
92 await conn.execute(
93 f"CREATE INDEX IF NOT EXISTS idx_{table}_data ON {table} USING GIN (data)"
94 )
95
96
97async def main():
98 if not DATABASE_URL:
99 print("ERROR: DATABASE_URL not set.")
100 print(" $env:DATABASE_URL='postgresql://...'")
101 sys.exit(1)
102
103 log.info("=" * 60)
104 log.info("BookARide: Bookaride_db → Neon Migration")
105 log.info(f"Mode: {'DRY RUN (add --confirm to execute)' if DRY_RUN else 'LIVE'}")
106 log.info("=" * 60)
107
108 # ── Connect to MongoDB ────────────────────────────────────────────────────
109 log.info("\nConnecting to MongoDB Bookaride_db...")
110 try:
111 mongo_client = pymongo.MongoClient(MONGO_URL, serverSelectionTimeoutMS=10000)
112 mongo_db = mongo_client.get_database()
113 mongo_client.admin.command("ping")
114 db_name = mongo_db.name
115 log.info(f" Connected to MongoDB database: {db_name}")
116 except Exception as e:
117 log.error(f" MongoDB connection failed: {e}")
118 sys.exit(1)
119
120 # ── Connect to Neon ───────────────────────────────────────────────────────
121 log.info("Connecting to Neon PostgreSQL...")
122 try:
123 pg = await asyncpg.connect(DATABASE_URL)
124 log.info(" Connected to Neon")
125 except Exception as e:
126 log.error(f" Neon connection failed: {e}")
127 sys.exit(1)
128
129 try:
130 # ── Step 1: Count what's coming ───────────────────────────────────────
131 log.info("\n── What's in Bookaride_db ──────────────────────────────")
132 total_to_import = 0
133 for mongo_col, _, desc in COLLECTIONS:
134 try:
135 count = mongo_db[mongo_col].count_documents({})
136 if count > 0:
137 log.info(f" {mongo_col:<30} {count:>6} docs ({desc})")
138 total_to_import += count
139 except Exception:
140 pass
141 log.info(f"\n Total to import: {total_to_import}")
142
143 # ── Step 2: Clean seeded test bookings from Neon ──────────────────────
144 log.info("\n── Step 1: Clean seeded test data from Neon ────────────")
145 try:
146 seeded = await pg.fetchval("""
147 SELECT COUNT(*) FROM bookings
148 WHERE (data->>'referenceNumber')::int BETWEEN 10 AND 1209
149 """)
150 log.info(f" Seeded test bookings found: {seeded}")
151
152 if not DRY_RUN and seeded > 0:
153 deleted = await pg.execute("""
154 DELETE FROM bookings
155 WHERE (data->>'referenceNumber')::int BETWEEN 10 AND 1209
156 """)
157 log.info(f" Deleted {deleted.split()[-1]} seeded bookings")
158 elif DRY_RUN:
159 log.info(f" (dry run) Would delete {seeded} seeded bookings")
160 except Exception as e:
161 log.warning(f" Could not clean seeded data: {e}")
162
163 # ── Step 3: Import each collection ────────────────────────────────────
164 log.info("\n── Step 2: Import from Bookaride_db ────────────────────")
165 grand_total = 0
166
167 for mongo_col, pg_table, desc in COLLECTIONS:
168 docs = list(mongo_db[mongo_col].find({}))
169 if not docs:
170 continue
171
172 log.info(f"\n {mongo_col} → {pg_table} ({len(docs)} docs)")
173
174 if DRY_RUN:
175 log.info(f" (dry run) Would import {len(docs)} documents")
176 continue
177
178 # Ensure table exists
179 await ensure_table(pg, pg_table)
180
181 inserted = 0
182 skipped = 0
183
184 for doc in docs:
185 clean = clean_doc(doc)
186 doc_id = (
187 clean.get("id") or
188 clean.get("bookingId") or
189 clean.get("referenceNumber") or
190 clean.get("email") or
191 None
192 )
193 if doc_id:
194 doc_id = str(doc_id)
195
196 data_json = json.dumps(clean)
197
198 try:
199 await pg.execute(
200 f"INSERT INTO {pg_table} (id, data) VALUES ($1, $2::jsonb) "
201 f"ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data",
202 doc_id, data_json
203 )
204 inserted += 1
205 except asyncpg.UniqueViolationError:
206 # No unique id — insert without id
207 try:
208 await pg.execute(
209 f"INSERT INTO {pg_table} (data) VALUES ($1::jsonb)",
210 data_json
211 )
212 inserted += 1
213 except Exception as e2:
214 log.warning(f" skip (error): {e2}")
215 skipped += 1
216 except Exception as e:
217 log.warning(f" skip: {e}")
218 skipped += 1
219
220 log.info(f" Imported {inserted}/{len(docs)} (skipped {skipped})")
221 grand_total += inserted
222
223 # ── Step 4: Final summary ─────────────────────────────────────────────
224 log.info("\n── Final Neon State ────────────────────────────────────")
225 tables = await pg.fetch(
226 "SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename"
227 )
228 for row in tables:
229 t = row["tablename"]
230 c = await pg.fetchval(f"SELECT COUNT(*) FROM {t}")
231 log.info(f" {t:<35} {c:>6} rows")
232
233 log.info("\n" + "=" * 60)
234 if DRY_RUN:
235 log.info("DRY RUN complete — no changes were made.")
236 log.info("Run with --confirm to execute the migration.")
237 else:
238 log.info(f"Migration complete! Imported {grand_total} documents.")
239 log.info("\nNext steps:")
240 log.info(" 1. Run: python scripts\\verify_neon_data.py")
241 log.info(" 2. Test V2 admin dashboard and booking flow")
242 log.info(" 3. Update V2 env vars (DATABASE_URL in Render/Vercel)")
243 log.info(" 4. Once confirmed, decommission V1 and MongoDB Atlas")
244 log.info("=" * 60)
245
246 finally:
247 await pg.close()
248 mongo_client.close()
249
250
251if __name__ == "__main__":
252 asyncio.run(main())
0253
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts