diff --git a/apps/ops/src/composables/useSubscriptionActions.js b/apps/ops/src/composables/useSubscriptionActions.js index a4fc788..47870d0 100644 --- a/apps/ops/src/composables/useSubscriptionActions.js +++ b/apps/ops/src/composables/useSubscriptionActions.js @@ -8,12 +8,32 @@ import { authFetch } from 'src/api/auth' import { BASE_URL } from 'src/config/erpnext' import { formatMoney } from 'src/composables/useFormatters' -// Frappe REST update for Subscription doctype +// Frappe REST update — supports both Service Subscription (legacy) and native Subscription async function updateSub (name, fields) { - const res = await authFetch(BASE_URL + '/api/resource/Subscription/' + encodeURIComponent(name), { + // Detect which doctype based on name pattern: SUB-* = Service Subscription, ASUB-* = native Subscription + const doctype = name.startsWith('ASUB-') ? 'Subscription' : 'Service Subscription' + + // Map normalized field names back to target doctype fields + const mapped = {} + for (const [k, v] of Object.entries(fields)) { + if (doctype === 'Service Subscription') { + // Map template field names → Service Subscription field names + if (k === 'actual_price') mapped.monthly_price = v + else if (k === 'custom_description') mapped.plan_name = v + else if (k === 'billing_frequency') mapped.billing_cycle = v === 'A' ? 'Annuel' : 'Mensuel' + else if (k === 'status') mapped.status = v === 'Active' ? 'Actif' : v === 'Cancelled' ? 'Annulé' : v + else if (k === 'cancelation_date') mapped.cancellation_date = v + else if (k === 'cancel_at_period_end') { /* not applicable to Service Subscription */ } + else mapped[k] = v + } else { + mapped[k] = v + } + } + + const res = await authFetch(BASE_URL + '/api/resource/' + encodeURIComponent(doctype) + '/' + encodeURIComponent(name), { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(fields), + body: JSON.stringify(mapped), }) if (!res.ok) { const err = await res.text() diff --git a/apps/ops/src/composables/useSubscriptionGroups.js b/apps/ops/src/composables/useSubscriptionGroups.js index f500283..3ea86a4 100644 --- a/apps/ops/src/composables/useSubscriptionGroups.js +++ b/apps/ops/src/composables/useSubscriptionGroups.js @@ -22,33 +22,50 @@ export function subSection (sub) { const price = parseFloat(sub.actual_price || 0) const group = (sub.item_group || '').trim() const sku = (sub.item_code || '').toUpperCase() + const label = (sub.custom_description || sub.item_name || sub.plan_name || '').toLowerCase() - if (price < 0 || sku.startsWith('RAB')) return 'Rabais' + // Negative price = rabais + if (price < 0 || sku.startsWith('RAB') || label.includes('rabais')) return 'Rabais' + // Match by item_group / service_category for (const [section, cfg] of Object.entries(SECTION_MAP)) { if (section === 'Rabais') continue if (cfg.match.some(m => group.includes(m))) return section } - // SKU-based fallback + // Category-based matching (from Service Subscription.service_category) + if (group === 'Internet') return 'Internet' + if (group === 'VoIP' || group === 'Téléphonie') return 'Téléphonie' + if (group === 'IPTV' || group === 'Télévision') return 'Télévision' + if (group === 'Hébergement') return 'Hébergement' + if (group === 'Bundle') return 'Internet' + + // SKU-based fallback (works for both legacy item_codes and SUB-* names) if (/^(FTTH|FTTB|TURBO|FIBRE|FORFERF|FORFBASE|FORFPERF|FORFPOP|FORFMES|SYM|VIP|ENT|COM|FIBCOM|HV)/.test(sku)) return 'Internet' if (/^(TELEP|FAX|SERV911|SERVTEL|TELE_)/.test(sku)) return 'Téléphonie' if (/^(TV|STB|RABTV)/.test(sku)) return 'Télévision' if (/^(LOC|FTT_H|FTTH_LOC)/.test(sku)) return 'Équipement' if (/^(HEB|DOM)/.test(sku)) return 'Hébergement' + // Label-based fallback (Service Subscription plan_name) + if (/fibre|internet|giga|illimit|turbo|optimum/i.test(label)) return 'Internet' + if (/téléphon|voip|phone|fax|911/i.test(label)) return 'Téléphonie' + if (/télé|tv|décodeur|stb/i.test(label)) return 'Télévision' + if (/location|modem|routeur|appareil|frais.*accès/i.test(label)) return 'Équipement' + if (/héberg|domaine|cloud|espace/i.test(label)) return 'Hébergement' + return 'Autre' } export function isRebate (sub) { - return parseFloat(sub.actual_price || 0) < 0 + return parseFloat(sub.actual_price || sub.monthly_price || 0) < 0 } export function subMainLabel (sub) { if (sub.custom_description && sub.custom_description.trim()) { return sub.custom_description } - return sub.item_name || sub.item_code || sub.name + return sub.item_name || sub.plan_name || sub.item_code || sub.name } export function subSubLabel (sub) { diff --git a/apps/ops/src/pages/ClientDetailPage.vue b/apps/ops/src/pages/ClientDetailPage.vue index 00e2cee..e675005 100644 --- a/apps/ops/src/pages/ClientDetailPage.vue +++ b/apps/ops/src/pages/ClientDetailPage.vue @@ -126,9 +126,9 @@
-
+
- {{ sub.item_code || '—' }} + {{ sub.name }} {{ subMainLabel(sub) }} @@ -172,8 +172,8 @@ class="sub-row clickable-row" :class="{ 'sub-rebate': isRebate(sub), 'sub-cancelled': sub.status === 'Cancelled' }">
-
- {{ sub.item_code || '—' }} +
+ {{ sub.name }} A {{ subMainLabel(sub) }} @@ -600,13 +600,26 @@ async function loadCustomer (id) { 'longitude', 'latitude'], limit: 100, orderBy: 'status asc, address_line asc', }), - listDocs('Subscription', { - filters: partyFilter, - fields: ['name', 'status', 'start_date', 'service_location', 'radius_user', - 'actual_price', 'custom_description', 'item_code', 'item_group', 'billing_frequency', 'item_name', - 'cancel_at_period_end', 'current_invoice_start', 'current_invoice_end', 'end_date', 'cancelation_date'], - limit: 100, orderBy: 'start_date desc', - }), + listDocs('Service Subscription', { + filters: custFilter, + fields: ['name', 'status', 'start_date', 'end_date', 'service_location', + 'monthly_price', 'plan_name', 'service_category', 'billing_cycle', + 'speed_down', 'speed_up', 'cancellation_date', 'cancellation_reason', 'notes'], + limit: 200, orderBy: 'start_date desc', + }).then(subs => subs.map(s => ({ + // Normalize to match template field names (formerly from Subscription doctype) + ...s, + actual_price: s.monthly_price, + custom_description: s.plan_name, + item_name: s.plan_name, + item_code: s.name, + item_group: s.service_category || '', + billing_frequency: s.billing_cycle === 'Annuel' ? 'A' : 'M', + cancel_at_period_end: 0, + cancelation_date: s.cancellation_date, + // Map status: Actif→Active, Annulé→Cancelled, etc. + status: s.status === 'Actif' ? 'Active' : s.status === 'Annulé' ? 'Cancelled' : s.status === 'Suspendu' ? 'Cancelled' : s.status === 'En attente' ? 'Active' : s.status, + }))), listDocs('Service Equipment', { filters: custFilter, fields: ['name', 'equipment_type', 'brand', 'model', 'serial_number', 'mac_address', diff --git a/scripts/migration/reconcile_subscriptions.py b/scripts/migration/reconcile_subscriptions.py new file mode 100644 index 0000000..8fbf90f --- /dev/null +++ b/scripts/migration/reconcile_subscriptions.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python3 +""" +Reconcile Service Subscription (complete legacy import) vs Subscription (ERPNext native). + +Identifies per-customer discrepancies: + - Missing records in Subscription that exist in Service Subscription + - Price/status mismatches + - Generates a sync plan to fix the native Subscription table + +Run inside erpnext-backend-1: + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reconcile_subscriptions.py + +Modes: + --audit Audit only — report discrepancies (default) + --sync Create missing Subscriptions from Service Subscriptions + --fix Fix price/status mismatches on existing Subscriptions + --full Audit + Sync + Fix (do everything) +""" +import sys +import os +import json +from datetime import datetime, timezone +import uuid + +os.chdir("/home/frappe/frappe-bench/sites") + +import frappe +frappe.init(site="erp.gigafibre.ca", sites_path=".") +frappe.connect() +frappe.local.flags.ignore_permissions = True +print(f"Connected: {frappe.local.site}") + +ADMIN = "Administrator" +COMPANY = "TARGO" +TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T" + + +def uid(prefix=""): + return prefix + uuid.uuid4().hex[:10] + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 1: Load all Service Subscriptions (source of truth) +# ═══════════════════════════════════════════════════════════════ +def load_service_subscriptions(): + """Load all Service Subscriptions grouped by customer.""" + print("\n" + "=" * 70) + print("LOADING SERVICE SUBSCRIPTIONS (source of truth)") + print("=" * 70) + + rows = frappe.db.sql(""" + SELECT name, customer, service_location, plan_name, monthly_price, + status, service_category, billing_cycle, start_date, end_date, + speed_down, speed_up, cancellation_date, cancellation_reason, + legacy_service_id, product_sku, radius_user, radius_password + FROM "tabService Subscription" + ORDER BY customer, service_location, monthly_price DESC + """, as_dict=True) + + by_customer = {} + for r in rows: + cust = r["customer"] + if cust not in by_customer: + by_customer[cust] = [] + by_customer[cust].append(r) + + total = len(rows) + active = sum(1 for r in rows if r["status"] == "Actif") + customers = len(by_customer) + print(f" Total: {total} records, {active} active, {customers} customers") + return by_customer + + +# ═══════════════════════════════════════════════════════════════ +# STEP 2: Load all Subscriptions (ERPNext native) +# ═══════════════════════════════════════════════════════════════ +def load_subscriptions(): + """Load all native Subscriptions grouped by party (customer).""" + print("\n" + "=" * 70) + print("LOADING SUBSCRIPTIONS (ERPNext native)") + print("=" * 70) + + rows = frappe.db.sql(""" + SELECT s.name, s.party as customer, s.status, s.start_date, s.end_date, + s.actual_price, s.custom_description, s.item_code, s.item_group, + s.billing_frequency, s.legacy_service_id, s.service_location, + s.radius_user, s.cancel_at_period_end, s.cancelation_date, + s.additional_discount_amount + FROM "tabSubscription" s + ORDER BY s.party, s.service_location, s.actual_price DESC + """, as_dict=True) + + by_customer = {} + by_legacy_id = {} + for r in rows: + cust = r["customer"] + if cust not in by_customer: + by_customer[cust] = [] + by_customer[cust].append(r) + if r.get("legacy_service_id"): + by_legacy_id[r["legacy_service_id"]] = r + + total = len(rows) + active = sum(1 for r in rows if r["status"] == "Active") + customers = len(by_customer) + print(f" Total: {total} records, {active} active, {customers} customers") + return by_customer, by_legacy_id + + +# ═══════════════════════════════════════════════════════════════ +# STEP 3: Audit — Compare per-customer +# ═══════════════════════════════════════════════════════════════ +def audit(ss_by_cust, sub_by_cust, sub_by_legacy_id): + """Compare per-customer and identify discrepancies.""" + print("\n" + "=" * 70) + print("AUDIT: COMPARING PER-CUSTOMER") + print("=" * 70) + + all_customers = set(list(ss_by_cust.keys()) + list(sub_by_cust.keys())) + + # Discrepancy tracking + missing_in_sub = [] # Service Subscriptions with no matching Subscription + price_mismatches = [] # Matching records with different prices + status_mismatches = [] # Matching records with different statuses + count_mismatches = [] # Customers with different record counts + total_mismatches = [] # Customers with different monthly totals + + customers_ok = 0 + customers_mismatch = 0 + + for cust in sorted(all_customers): + ss_list = ss_by_cust.get(cust, []) + sub_list = sub_by_cust.get(cust, []) + + # Compare totals + ss_total = sum(float(s.get("monthly_price") or 0) for s in ss_list if s["status"] == "Actif") + sub_total = sum(float(s.get("actual_price") or 0) for s in sub_list if s["status"] == "Active") + + ss_active = [s for s in ss_list if s["status"] == "Actif"] + sub_active = [s for s in sub_list if s["status"] == "Active"] + + has_issue = False + + # Count mismatch + if len(ss_active) != len(sub_active): + count_mismatches.append({ + "customer": cust, + "ss_count": len(ss_active), + "sub_count": len(sub_active), + "diff": len(ss_active) - len(sub_active), + }) + has_issue = True + + # Total mismatch (more than 0.01$ difference) + if abs(ss_total - sub_total) > 0.01: + total_mismatches.append({ + "customer": cust, + "ss_total": ss_total, + "sub_total": sub_total, + "diff": ss_total - sub_total, + }) + has_issue = True + + # Per-record matching via legacy_service_id + for ss in ss_list: + legacy_id = ss.get("legacy_service_id") + if legacy_id and legacy_id in sub_by_legacy_id: + sub = sub_by_legacy_id[legacy_id] + # Price check + ss_price = float(ss.get("monthly_price") or 0) + sub_price = float(sub.get("actual_price") or 0) + if abs(ss_price - sub_price) > 0.01: + price_mismatches.append({ + "customer": cust, + "ss_name": ss["name"], + "sub_name": sub["name"], + "legacy_id": legacy_id, + "plan": ss.get("plan_name") or "", + "ss_price": ss_price, + "sub_price": sub_price, + "diff": ss_price - sub_price, + }) + has_issue = True + + # Status check + ss_status = "Active" if ss["status"] == "Actif" else "Cancelled" + sub_status = sub["status"] + if ss_status != sub_status: + status_mismatches.append({ + "customer": cust, + "ss_name": ss["name"], + "sub_name": sub["name"], + "legacy_id": legacy_id, + "ss_status": ss["status"], + "sub_status": sub_status, + }) + has_issue = True + elif legacy_id: + # No matching Subscription found + missing_in_sub.append({ + "customer": cust, + "ss_name": ss["name"], + "legacy_id": legacy_id, + "plan_name": ss.get("plan_name") or "", + "monthly_price": float(ss.get("monthly_price") or 0), + "status": ss["status"], + "service_location": ss.get("service_location") or "", + "service_category": ss.get("service_category") or "", + "billing_cycle": ss.get("billing_cycle") or "Mensuel", + "start_date": ss.get("start_date"), + "product_sku": ss.get("product_sku") or "", + "radius_user": ss.get("radius_user") or "", + "radius_password": ss.get("radius_password") or "", + }) + has_issue = True + + if has_issue: + customers_mismatch += 1 + else: + customers_ok += 1 + + # ── Report ── + print(f"\n Customers OK (totals match): {customers_ok}") + print(f" Customers with discrepancies: {customers_mismatch}") + print(f" ---") + print(f" Missing in Subscription: {len(missing_in_sub)}") + print(f" Price mismatches: {len(price_mismatches)}") + print(f" Status mismatches: {len(status_mismatches)}") + print(f" Count mismatches: {len(count_mismatches)}") + print(f" Total (monthly $) mismatches: {len(total_mismatches)}") + + # Top 20 total mismatches + if total_mismatches: + print(f"\n TOP 20 MONTHLY TOTAL MISMATCHES:") + sorted_tm = sorted(total_mismatches, key=lambda x: abs(x["diff"]), reverse=True) + for tm in sorted_tm[:20]: + print(f" {tm['customer']:25s} SS: {tm['ss_total']:>8.2f} SUB: {tm['sub_total']:>8.2f} DIFF: {tm['diff']:>+8.2f}") + + # Top 20 count mismatches + if count_mismatches: + print(f"\n TOP 20 COUNT MISMATCHES:") + sorted_cm = sorted(count_mismatches, key=lambda x: abs(x["diff"]), reverse=True) + for cm in sorted_cm[:20]: + print(f" {cm['customer']:25s} SS: {cm['ss_count']:3d} SUB: {cm['sub_count']:3d} DIFF: {cm['diff']:+3d}") + + # Missing by category + if missing_in_sub: + cats = {} + for m in missing_in_sub: + cat = m["service_category"] or "Unknown" + cats[cat] = cats.get(cat, 0) + 1 + print(f"\n MISSING IN SUBSCRIPTION BY CATEGORY:") + for cat, count in sorted(cats.items(), key=lambda x: -x[1]): + print(f" {cat:25s} {count:5d}") + + return { + "missing": missing_in_sub, + "price_mismatches": price_mismatches, + "status_mismatches": status_mismatches, + "count_mismatches": count_mismatches, + "total_mismatches": total_mismatches, + } + + +# ═══════════════════════════════════════════════════════════════ +# STEP 4: Sync — Create missing Subscriptions +# ═══════════════════════════════════════════════════════════════ +def sync_missing(missing_records): + """Create native Subscriptions for records missing from the Subscription table.""" + print("\n" + "=" * 70) + print(f"SYNC: CREATING {len(missing_records)} MISSING SUBSCRIPTIONS") + print("=" * 70) + + if not missing_records: + print(" Nothing to sync!") + return + + ts = now() + + # Load existing subscription plans + plans = frappe.db.sql('SELECT plan_name, name FROM "tabSubscription Plan"', as_dict=True) + plan_map = {p["plan_name"]: p["name"] for p in plans} + + # Load existing items (to find or create plans) + items = frappe.db.sql('SELECT name, item_name, item_group FROM "tabItem"', as_dict=True) + item_map = {i["name"]: i for i in items} + + created = 0 + errors = 0 + plan_created = 0 + + for rec in missing_records: + sku = rec["product_sku"] or "UNKNOWN" + plan_name = f"PLAN-{sku}" + + # Create plan if it doesn't exist + if plan_name not in plan_map: + plan_id = uid("SP-") + try: + frappe.db.sql(""" + INSERT INTO "tabSubscription Plan" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan_name, item, currency, price_determination, cost, + billing_interval, billing_interval_count + ) VALUES (%s, %s, %s, %s, %s, 0, 0, + %s, %s, 'CAD', 'Fixed Rate', %s, %s, 1) + """, (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, + abs(rec["monthly_price"]), + "Year" if rec["billing_cycle"] == "Annuel" else "Month")) + plan_map[plan_name] = plan_id + plan_created += 1 + except Exception as e: + frappe.db.rollback() + print(f" ERR plan {plan_name}: {str(e)[:80]}") + + # Determine status + status = "Active" if rec["status"] == "Actif" else "Cancelled" + + # Determine discount: negative price = discount record, handle differently + price = rec["monthly_price"] + discount = 0 + if price < 0: + # This is a rebate/discount line — store as negative actual_price + discount = 0 # Don't use additional_discount for negative-price lines + + # Start date + start_date = rec.get("start_date") or "2020-01-01" + if hasattr(start_date, 'strftime'): + start_date = start_date.strftime("%Y-%m-%d") + + # Billing frequency + billing_freq = "A" if rec["billing_cycle"] == "Annuel" else "M" + + sub_id = uid("SUB-") + try: + frappe.db.sql(""" + INSERT INTO "tabSubscription" ( + name, creation, modified, modified_by, owner, docstatus, idx, + party_type, party, company, status, + start_date, generate_invoice_at, days_until_due, + follow_calendar_months, generate_new_invoices_past_due_date, + submit_invoice, cancel_at_period_end, + sales_tax_template, + additional_discount_amount, + radius_user, radius_pwd, legacy_service_id, + service_location, + actual_price, custom_description, item_code, item_group, + billing_frequency + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + 'Customer', %s, %s, %s, + %s, 'Beginning of the current subscription period', 30, + 0, 1, 0, 0, + %s, %s, + %s, %s, %s, + %s, + %s, %s, %s, %s, %s + ) + """, (sub_id, ts, ts, ADMIN, ADMIN, + rec["customer"], COMPANY, status, + start_date, + TAX_TEMPLATE, discount, + rec.get("radius_user") or None, + rec.get("radius_password") or None, + rec["legacy_id"], + rec.get("service_location") or None, + price, rec.get("plan_name") or "", + sku, rec.get("service_category") or "", + billing_freq)) + + # Create plan detail child record + if plan_name in plan_map: + frappe.db.sql(""" + INSERT INTO "tabSubscription Plan Detail" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan, qty, + parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, 1, + %s, 1, + %s, 'plans', 'Subscription') + """, (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id)) + + created += 1 + + if created % 500 == 0: + frappe.db.commit() + print(f" [{created}/{len(missing_records)}] created...") + + except Exception as e: + errors += 1 + frappe.db.rollback() + if errors <= 20: + print(f" ERR {rec['customer']} {rec['ss_name']}: {str(e)[:100]}") + + frappe.db.commit() + print(f"\n Plans created: {plan_created}") + print(f" Subscriptions created: {created}") + print(f" Errors: {errors}") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 5: Fix — Correct price/status mismatches +# ═══════════════════════════════════════════════════════════════ +def fix_mismatches(price_mismatches, status_mismatches): + """Fix price and status mismatches on existing Subscriptions.""" + print("\n" + "=" * 70) + print(f"FIX: CORRECTING {len(price_mismatches)} PRICE + {len(status_mismatches)} STATUS MISMATCHES") + print("=" * 70) + + fixed_price = 0 + fixed_status = 0 + + for pm in price_mismatches: + try: + frappe.db.sql(""" + UPDATE "tabSubscription" + SET actual_price = %s, modified = %s + WHERE name = %s + """, (pm["ss_price"], now(), pm["sub_name"])) + fixed_price += 1 + except Exception as e: + print(f" ERR price fix {pm['sub_name']}: {str(e)[:80]}") + + for sm in status_mismatches: + new_status = "Active" if sm["ss_status"] == "Actif" else "Cancelled" + try: + frappe.db.sql(""" + UPDATE "tabSubscription" + SET status = %s, modified = %s + WHERE name = %s + """, (new_status, now(), sm["sub_name"])) + fixed_status += 1 + except Exception as e: + print(f" ERR status fix {sm['sub_name']}: {str(e)[:80]}") + + frappe.db.commit() + print(f" Price fixes: {fixed_price}") + print(f" Status fixes: {fixed_status}") + + +# ═══════════════════════════════════════════════════════════════ +# MAIN +# ═══════════════════════════════════════════════════════════════ +def main(): + mode = sys.argv[1] if len(sys.argv) > 1 else "--audit" + + print(f"\nMode: {mode}") + print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + + # Load data + ss_by_cust = load_service_subscriptions() + sub_by_cust, sub_by_legacy_id = load_subscriptions() + + # Audit + results = audit(ss_by_cust, sub_by_cust, sub_by_legacy_id) + + if mode in ("--sync", "--full"): + sync_missing(results["missing"]) + + if mode in ("--fix", "--full"): + fix_mismatches(results["price_mismatches"], results["status_mismatches"]) + + # Write report file + report_path = "/tmp/reconcile_report.json" + report = { + "timestamp": datetime.now().isoformat(), + "mode": mode, + "summary": { + "missing_in_subscription": len(results["missing"]), + "price_mismatches": len(results["price_mismatches"]), + "status_mismatches": len(results["status_mismatches"]), + "count_mismatches": len(results["count_mismatches"]), + "total_mismatches": len(results["total_mismatches"]), + }, + "total_mismatches_top50": sorted(results["total_mismatches"], key=lambda x: abs(x["diff"]), reverse=True)[:50], + "missing_sample": results["missing"][:50], + } + with open(report_path, "w") as f: + json.dump(report, f, indent=2, default=str) + print(f"\nReport saved: {report_path}") + + frappe.clear_cache() + print("\nDone — cache cleared") + + +if __name__ == "__main__": + main() diff --git a/scripts/migration/reimport_subscriptions.py b/scripts/migration/reimport_subscriptions.py new file mode 100644 index 0000000..495bcc4 --- /dev/null +++ b/scripts/migration/reimport_subscriptions.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +""" +Clean reimport of native Subscriptions from Service Subscription (source of truth). + +Fixes the migration issue where Phase 3/4 created hex customer IDs (CUST-04ea550dcf) +instead of mapping to the correct sequential IDs (CUST-988). + +Steps: + 1. Purge all native Subscriptions + Plan Details + 2. Recreate Subscriptions from Service Subscription data (correct customer mapping) + 3. Delete orphaned hex customers (no remaining FK references) + 4. Verify totals match + +Run inside erpnext-backend-1: + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reimport_subscriptions.py [--dry-run] + +Log: stdout (run with tee) +""" +import sys +import os +import uuid +import time +from datetime import datetime, timezone + +os.chdir("/home/frappe/frappe-bench/sites") +import frappe + +frappe.init(site="erp.gigafibre.ca", sites_path=".") +frappe.connect() +frappe.local.flags.ignore_permissions = True +print(f"Connected: {frappe.local.site}") + +DRY_RUN = "--dry-run" in sys.argv +if DRY_RUN: + print("*** DRY RUN — no changes will be written ***") + +ADMIN = "Administrator" +COMPANY = "TARGO" +TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T" + + +def uid(prefix=""): + return prefix + uuid.uuid4().hex[:10] + + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 1: PURGE native Subscriptions +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 1: PURGE EXISTING SUBSCRIPTIONS") +print("=" * 70) + +sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0] +spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0] +sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0] + +print(f" Current counts: {sub_count} Subscriptions, {spd_count} Plan Details, {sp_count} Plans") + +if not DRY_RUN: + frappe.db.sql('DELETE FROM "tabSubscription Plan Detail"') + frappe.db.sql('DELETE FROM "tabSubscription"') + frappe.db.sql('DELETE FROM "tabSubscription Plan"') + frappe.db.commit() + print(" Purged all Subscriptions, Plan Details, and Plans.") +else: + print(" [DRY RUN] Would purge all Subscriptions, Plan Details, and Plans.") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 2: LOAD Service Subscriptions (source of truth) +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 2: LOAD SERVICE SUBSCRIPTIONS") +print("=" * 70) + +ss_rows = frappe.db.sql(""" + SELECT name, customer, service_location, plan_name, monthly_price, + status, service_category, billing_cycle, start_date, end_date, + speed_down, speed_up, cancellation_date, cancellation_reason, + legacy_service_id, product_sku, radius_user, radius_password, notes + FROM "tabService Subscription" + ORDER BY customer, service_location, monthly_price DESC +""", as_dict=True) + +total = len(ss_rows) +active = sum(1 for r in ss_rows if r["status"] == "Actif") +customers = len(set(r["customer"] for r in ss_rows)) +print(f" Loaded {total} Service Subscriptions ({active} active) for {customers} customers") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 3: BUILD Subscription Plans from unique SKUs +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 3: CREATE SUBSCRIPTION PLANS") +print("=" * 70) +t0 = time.time() + +# Collect unique SKUs with their typical price and category +sku_info = {} +for r in ss_rows: + sku = r.get("product_sku") or "" + if not sku: + continue + if sku not in sku_info: + sku_info[sku] = { + "price": abs(float(r["monthly_price"] or 0)), + "category": r.get("service_category") or "", + "billing_cycle": r.get("billing_cycle") or "Mensuel", + "plan_name": r.get("plan_name") or sku, + } + +# Also add a fallback plan for records without SKU +plan_map = {} # plan_name → plan_id +plans_created = 0 + +if not DRY_RUN: + ts = now() + for sku, info in sku_info.items(): + plan_name = f"PLAN-{sku}" + plan_id = uid("SP-") + billing_interval = "Year" if info["billing_cycle"] == "Annuel" else "Month" + + try: + frappe.db.sql(""" + INSERT INTO "tabSubscription Plan" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan_name, item, currency, price_determination, cost, + billing_interval, billing_interval_count + ) VALUES (%s, %s, %s, %s, %s, 0, 0, + %s, %s, 'CAD', 'Fixed Rate', %s, %s, 1) + """, (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, info["price"], billing_interval)) + plan_map[plan_name] = plan_id + plans_created += 1 + except Exception as e: + frappe.db.rollback() + print(f" ERR plan {sku}: {str(e)[:80]}") + + frappe.db.commit() + +print(f" Created {plans_created} Subscription Plans [{time.time()-t0:.1f}s]") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 4: CREATE Subscriptions from Service Subscriptions +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 4: CREATE SUBSCRIPTIONS") +print("=" * 70) +t0 = time.time() + +# Verify all customers exist +all_customers = set(r["customer"] for r in ss_rows) +existing_customers = set( + r[0] for r in frappe.db.sql('SELECT name FROM "tabCustomer"') +) +missing_customers = all_customers - existing_customers +if missing_customers: + print(f" WARNING: {len(missing_customers)} customers referenced but don't exist!") + for c in list(missing_customers)[:10]: + print(f" {c}") + +sub_ok = sub_err = 0 +ts = now() + +for i, ss in enumerate(ss_rows): + if DRY_RUN: + sub_ok += 1 + continue + + cust = ss["customer"] + if cust in missing_customers: + sub_err += 1 + continue + + # Map status + if ss["status"] == "Actif": + status = "Active" + elif ss["status"] == "Annulé": + status = "Cancelled" + elif ss["status"] == "Suspendu": + status = "Past Due Date" + else: + status = "Active" # default + + # Price + price = float(ss["monthly_price"] or 0) + + # Start date + start_date = ss.get("start_date") or "2020-01-01" + if hasattr(start_date, "strftime"): + start_date = start_date.strftime("%Y-%m-%d") + + # End/cancellation date + end_date = ss.get("end_date") + if hasattr(end_date, "strftime"): + end_date = end_date.strftime("%Y-%m-%d") + cancel_date = ss.get("cancellation_date") + if hasattr(cancel_date, "strftime"): + cancel_date = cancel_date.strftime("%Y-%m-%d") + + # Billing frequency + billing_freq = "A" if ss.get("billing_cycle") == "Annuel" else "M" + + # SKU / plan + sku = ss.get("product_sku") or "" + plan_name = f"PLAN-{sku}" if sku else "" + + # Category + category = ss.get("service_category") or "" + + # Description: plan_name from SS + description = ss.get("plan_name") or "" + + sub_id = uid("SUB-") + + try: + frappe.db.sql(""" + INSERT INTO "tabSubscription" ( + name, creation, modified, modified_by, owner, docstatus, idx, + party_type, party, company, status, + start_date, generate_invoice_at, days_until_due, + follow_calendar_months, generate_new_invoices_past_due_date, + submit_invoice, cancel_at_period_end, + sales_tax_template, + additional_discount_amount, + radius_user, radius_pwd, legacy_service_id, + service_location, + actual_price, custom_description, item_code, item_group, + billing_frequency, + cancelation_date + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + 'Customer', %s, %s, %s, + %s, 'Beginning of the current subscription period', 30, + 1, 1, 0, 0, + %s, 0, + %s, %s, %s, + %s, + %s, %s, %s, %s, %s, + %s + ) + """, (sub_id, ts, ts, ADMIN, ADMIN, + cust, COMPANY, status, + start_date, + TAX_TEMPLATE, + ss.get("radius_user") or None, + ss.get("radius_password") or None, + ss.get("legacy_service_id"), + ss.get("service_location") or None, + price, description, sku, category, billing_freq, + cancel_date)) + + # Create plan detail child record + if plan_name and plan_name in plan_map: + frappe.db.sql(""" + INSERT INTO "tabSubscription Plan Detail" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan, qty, + parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, 1, + %s, 1, + %s, 'plans', 'Subscription') + """, (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id)) + + sub_ok += 1 + + if sub_ok % 2000 == 0: + frappe.db.commit() + print(f" [{sub_ok}/{total}] created... [{time.time()-t0:.0f}s]") + + except Exception as e: + sub_err += 1 + frappe.db.rollback() + if sub_err <= 20: + print(f" ERR {cust}/{ss['name']}: {str(e)[:100]}") + +if not DRY_RUN: + frappe.db.commit() + +print(f" Created {sub_ok} Subscriptions, {sub_err} errors [{time.time()-t0:.1f}s]") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 5: CLEANUP — Delete orphaned hex customers +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 5: CLEANUP ORPHANED HEX CUSTOMERS") +print("=" * 70) +t0 = time.time() + +# Find hex-pattern customers (non-numeric suffix after CUST-) +hex_customers = frappe.db.sql(""" + SELECT name, customer_name, legacy_account_id + FROM "tabCustomer" + WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$' +""", as_dict=True) + +print(f" Found {len(hex_customers)} hex-pattern customers") + +if hex_customers: + # Check which hex customers still have FK references using subqueries (avoid ANY() limits) + hex_subquery = """(SELECT name FROM "tabCustomer" WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$')""" + + fk_tables = [ + ("tabSales Invoice", "customer"), + ("tabPayment Entry", "party"), + ("tabIssue", "customer"), + ("tabService Location", "customer"), + ("tabService Subscription", "customer"), + ("tabSubscription", "party"), + ("tabService Equipment", "customer"), + ] + + referenced = set() + for table, col in fk_tables: + try: + refs = frappe.db.sql( + f'SELECT DISTINCT "{col}" FROM "{table}" WHERE "{col}" IN {hex_subquery}' + ) + for r in refs: + referenced.add(r[0]) + print(f" {table:30s} {col:15s} → {len(refs)} hex refs") + except Exception as e: + frappe.db.rollback() + print(f" ERR {table}: {str(e)[:60]}") + + orphaned = [c for c in hex_customers if c["name"] not in referenced] + still_used = [c for c in hex_customers if c["name"] in referenced] + + print(f" Still referenced (keep): {len(still_used)}") + print(f" Orphaned (can delete): {len(orphaned)}") + + if orphaned and not DRY_RUN: + # Delete orphaned hex customers using subquery (avoid ANY() limits) + # First, create a temp table of referenced hex customers + frappe.db.sql(""" + DELETE FROM "tabDynamic Link" + WHERE link_doctype = 'Customer' + AND link_name IN (SELECT name FROM "tabCustomer" + WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$' + AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL) + ) + """) + frappe.db.commit() + + deleted = frappe.db.sql(""" + DELETE FROM "tabCustomer" + WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$' + AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL) + AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL) + """) + frappe.db.commit() + print(f" Deleted orphaned hex customers") + elif orphaned: + print(f" [DRY RUN] Would delete {len(orphaned)} orphaned hex customers") +else: + print(" No hex-pattern customers found.") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 6: VERIFY +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 6: VERIFICATION") +print("=" * 70) + +new_sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0] +new_spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0] +new_sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0] +cust_count = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer"')[0][0] + +print(f" Subscriptions: {new_sub_count}") +print(f" Plan Details: {new_spd_count}") +print(f" Plans: {new_sp_count}") +print(f" Customers: {cust_count}") + +# Check for customer ID consistency: all subscriptions should point to sequential customers +hex_refs = frappe.db.sql(""" + SELECT COUNT(*) FROM "tabSubscription" + WHERE party ~ '^CUST-' AND party !~ '^CUST-[0-9]+$' +""")[0][0] +print(f" Subscriptions pointing to hex customers: {hex_refs} (should be 0)") + +# Spot-check: LPB4 customer (legacy_account_id = 4) +lpb4 = frappe.db.sql(""" + SELECT name FROM "tabCustomer" WHERE legacy_account_id = 4 +""", as_dict=True) +if lpb4: + cust_name = lpb4[0]["name"] + lpb4_subs = frappe.db.sql(""" + SELECT actual_price, item_code, custom_description, status, billing_frequency + FROM "tabSubscription" + WHERE party = %s + ORDER BY actual_price DESC + """, (cust_name,), as_dict=True) + total_price = sum(float(s["actual_price"] or 0) for s in lpb4_subs if s["status"] == "Active") + print(f"\n Spot-check: {cust_name} (LPB4)") + for s in lpb4_subs: + print(f" {s['item_code'] or '':15s} {float(s['actual_price'] or 0):>8.2f} {s['billing_frequency']} {s['status']} {s['custom_description'] or ''}") + print(f" {'TOTAL':15s} {total_price:>8.2f}$/mo") + +# Summary +print("\n" + "=" * 70) +print("COMPLETE") +print("=" * 70) +print(f" Next: bench --site erp.gigafibre.ca clear-cache") +print() diff --git a/scripts/migration/rename_all_doctypes.py b/scripts/migration/rename_all_doctypes.py new file mode 100644 index 0000000..e3acf3e --- /dev/null +++ b/scripts/migration/rename_all_doctypes.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +""" +Rename all doctype IDs to zero-padded 10-digit numeric format. + +Format: {PREFIX}-{legacy_id:010d} + SINV-0000001234 (Sales Invoice) + PE-0000001234 (Payment Entry) + ISS-0000001234 (Issue) + LOC-0000001234 (Service Location) + EQP-0000001234 (Service Equipment) + SUB-0000001234 (Service Subscription) + +Two-phase rename to avoid PK collisions. +Uses temp mapping tables + bulk UPDATE ... FROM for FK references. + +Run inside erpnext-backend-1: + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/rename_all_doctypes.py [--dry-run] +""" +import sys +import os +import time + +os.chdir("/home/frappe/frappe-bench/sites") +import frappe + +frappe.init(site="erp.gigafibre.ca", sites_path=".") +frappe.connect() +frappe.local.flags.ignore_permissions = True +print(f"Connected: {frappe.local.site}") + +DRY_RUN = "--dry-run" in sys.argv +if DRY_RUN: + print("*** DRY RUN ***\n") + + +def rename_doctype(label, table, prefix, legacy_col, fk_refs, pad=10): + """ + Rename a doctype's records to {prefix}-{legacy_id:0{pad}d}. + + Args: + label: Human-readable label + table: PostgreSQL table name (e.g. "tabSales Invoice") + prefix: ID prefix (e.g. "SINV") + legacy_col: Column containing the legacy numeric ID + fk_refs: List of (fk_table, fk_column, extra_where) tuples + pad: Zero-padding width (default 10) + """ + print("\n" + "=" * 70) + print(f"RENAMING: {label} → {prefix}-{('0' * pad)}") + print("=" * 70) + t_start = time.time() + + # Step 1: Build mapping + rows = frappe.db.sql(f""" + SELECT name, {legacy_col} FROM "{table}" + WHERE {legacy_col} IS NOT NULL AND {legacy_col} > 0 + """, as_dict=True) + + mapping = {} + new_used = set() + collisions = 0 + for r in rows: + old_name = r["name"] + new_name = f"{prefix}-{int(r[legacy_col]):0{pad}d}" + if new_name in new_used: + collisions += 1 + new_name = f"{new_name}d{collisions}" + new_used.add(new_name) + if old_name != new_name: + mapping[old_name] = new_name + + # Also handle records WITHOUT legacy ID + no_legacy = frappe.db.sql(f""" + SELECT COUNT(*) FROM "{table}" + WHERE {legacy_col} IS NULL OR {legacy_col} = 0 + """)[0][0] + + total = len(rows) + no_legacy + print(f" Total records: {total}") + print(f" Will rename: {len(mapping)}") + print(f" Already correct: {len(rows) - len(mapping)}") + print(f" No legacy ID (skip): {no_legacy}") + if collisions: + print(f" Collisions: {collisions}") + + if not mapping: + print(" Nothing to rename!") + return + + # Show samples + samples = list(mapping.items())[:5] + for old, new in samples: + print(f" {old:30s} → {new}") + + if DRY_RUN: + return + + # Step 2: Create temp mapping table + map_table = f"_rename_map_{prefix.lower()}" + frappe.db.sql(f'DROP TABLE IF EXISTS {map_table}') + frappe.db.sql(f""" + CREATE TEMP TABLE {map_table} ( + old_name VARCHAR(140) PRIMARY KEY, + new_name VARCHAR(140) NOT NULL + ) + """) + + # Insert mappings in batches + items = list(mapping.items()) + batch_size = 1000 + for start in range(0, len(items), batch_size): + batch = items[start:start + batch_size] + frappe.db.sql( + f"INSERT INTO {map_table} (old_name, new_name) VALUES " + + ",".join(["(%s, %s)"] * len(batch)), + [v for pair in batch for v in pair] + ) + frappe.db.commit() + + # Step 3: Update FK references + for fk_table, fk_col, extra_where in fk_refs: + t0 = time.time() + try: + frappe.db.sql(f""" + UPDATE "{fk_table}" t + SET "{fk_col}" = m.new_name + FROM {map_table} m + WHERE t."{fk_col}" = m.old_name + {extra_where} + """) + frappe.db.commit() + print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]") + except Exception as e: + frappe.db.rollback() + err = str(e)[:80] + if "does not exist" not in err: + print(f" FK {fk_table}.{fk_col} ERR: {err}") + + # Step 4: Two-phase rename + # Phase A: old → _TEMP_new + print(f" Phase A: temp rename...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '{old}' THEN '_TEMP_{new}'" for old, new in batch) + old_names = "','".join(old for old, _ in batch) + frappe.db.sql(f""" + UPDATE "{table}" + SET name = CASE name {cases} END + WHERE name IN ('{old_names}') + """) + frappe.db.commit() + print(f" Phase A done [{time.time()-t0:.1f}s]") + + # Phase B: _TEMP_new → new + print(f" Phase B: final rename...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '_TEMP_{new}' THEN '{new}'" for _, new in batch) + temp_names = "','".join(f"_TEMP_{new}" for _, new in batch) + frappe.db.sql(f""" + UPDATE "{table}" + SET name = CASE name {cases} END + WHERE name IN ('{temp_names}') + """) + frappe.db.commit() + print(f" Phase B done [{time.time()-t0:.1f}s]") + + # Clean up + frappe.db.sql(f'DROP TABLE IF EXISTS {map_table}') + frappe.db.commit() + + elapsed = time.time() - t_start + print(f" ✓ {label} complete [{elapsed:.1f}s]") + + +def set_series(prefix, start_at): + """Set the naming series counter.""" + existing = frappe.db.sql( + 'SELECT current FROM "tabSeries" WHERE name = %s', (prefix,), as_dict=True + ) + if existing: + if existing[0]["current"] < start_at: + frappe.db.sql('UPDATE "tabSeries" SET current = %s WHERE name = %s', (start_at, prefix)) + print(f" Series {prefix} updated to {start_at}") + else: + print(f" Series {prefix} already at {existing[0]['current']}") + else: + frappe.db.sql('INSERT INTO "tabSeries" (name, current) VALUES (%s, %s)', (prefix, start_at)) + print(f" Series {prefix} created at {start_at}") + frappe.db.commit() + + +# ═══════════════════════════════════════════════════════════════ +# 1. SALES INVOICE +# ═══════════════════════════════════════════════════════════════ +# Current: SINV-1 through SINV-99999 (legacy invoice.id) +# The 'name' is already SINV-{id}, we just need to zero-pad +# Legacy column: the name itself encodes the ID. We need to extract it. +# Actually, let's check if there's a legacy column... + +# Sales invoices were imported as SINV-{legacy_id}. The name IS the legacy ref. +# We need to extract the number from the name and zero-pad. +# Approach: use a direct SQL rename since legacy_id is embedded in name. + +print("\n" + "=" * 70) +print("RENAMING: Sales Invoice → SINV-0000XXXXXX") +print("=" * 70) +t_start = time.time() + +if not DRY_RUN: + # Build mapping from current names + sinv_rows = frappe.db.sql(""" + SELECT name FROM "tabSales Invoice" WHERE name ~ '^SINV-[0-9]+$' + """) + sinv_map = {} + for r in sinv_rows: + old = r[0] + num = int(old.replace("SINV-", "")) + new = f"SINV-{num:010d}" + if old != new: + sinv_map[old] = new + + print(f" Will rename: {len(sinv_map)} invoices") + for old, new in list(sinv_map.items())[:5]: + print(f" {old:20s} → {new}") + + if sinv_map: + # Create temp mapping + frappe.db.sql('DROP TABLE IF EXISTS _rename_sinv') + frappe.db.sql(""" + CREATE TEMP TABLE _rename_sinv ( + old_name VARCHAR(140) PRIMARY KEY, + new_name VARCHAR(140) NOT NULL + ) + """) + items = list(sinv_map.items()) + for start in range(0, len(items), 1000): + batch = items[start:start + 1000] + frappe.db.sql( + "INSERT INTO _rename_sinv (old_name, new_name) VALUES " + + ",".join(["(%s, %s)"] * len(batch)), + [v for pair in batch for v in pair] + ) + frappe.db.commit() + + # Update FK refs + sinv_fks = [ + ("tabSales Invoice Item", "parent", ""), + ("tabSales Invoice Payment", "parent", ""), + ("tabSales Taxes and Charges", "parent", "AND t.parenttype = 'Sales Invoice'"), + ("tabGL Entry", "voucher_no", "AND t.voucher_type = 'Sales Invoice'"), + ("tabPayment Ledger Entry", "voucher_no", "AND t.voucher_type = 'Sales Invoice'"), + ("tabPayment Entry Reference", "reference_name", "AND t.reference_doctype = 'Sales Invoice'"), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Sales Invoice'"), + ("tabVersion", "docname", "AND t.ref_doctype = 'Sales Invoice'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Sales Invoice'"), + ] + for fk_table, fk_col, extra_where in sinv_fks: + t0 = time.time() + try: + frappe.db.sql(f""" + UPDATE "{fk_table}" t + SET "{fk_col}" = m.new_name + FROM _rename_sinv m + WHERE t."{fk_col}" = m.old_name + {extra_where} + """) + frappe.db.commit() + print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]") + except Exception as e: + frappe.db.rollback() + err = str(e)[:80] + if "does not exist" not in err: + print(f" FK {fk_table}.{fk_col} ERR: {err}") + + # Two-phase rename + print(" Phase A...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '{old}' THEN '_T_{new}'" for old, new in batch) + old_names = "','".join(old for old, _ in batch) + frappe.db.sql(f""" + UPDATE "tabSales Invoice" + SET name = CASE name {cases} END + WHERE name IN ('{old_names}') + """) + if (start + 500) % 10000 < 500: + frappe.db.commit() + print(f" {min(start + 500, len(items))}/{len(items)}") + frappe.db.commit() + print(f" Phase A done [{time.time()-t0:.1f}s]") + + print(" Phase B...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '_T_{new}' THEN '{new}'" for _, new in batch) + temp_names = "','".join(f"_T_{new}" for _, new in batch) + frappe.db.sql(f""" + UPDATE "tabSales Invoice" + SET name = CASE name {cases} END + WHERE name IN ('{temp_names}') + """) + if (start + 500) % 10000 < 500: + frappe.db.commit() + print(f" {min(start + 500, len(items))}/{len(items)}") + frappe.db.commit() + print(f" Phase B done [{time.time()-t0:.1f}s]") + + frappe.db.sql('DROP TABLE IF EXISTS _rename_sinv') + frappe.db.commit() + + print(f" ✓ Sales Invoice [{time.time()-t_start:.1f}s]") +else: + sinv_count = frappe.db.sql("SELECT COUNT(*) FROM \"tabSales Invoice\" WHERE name ~ '^SINV-[0-9]+$'")[0][0] + print(f" [DRY] Would rename ~{sinv_count} invoices") + + +# ═══════════════════════════════════════════════════════════════ +# 2. PAYMENT ENTRY — same pattern as SINV +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("RENAMING: Payment Entry → PE-0000XXXXXX") +print("=" * 70) +t_start = time.time() + +if not DRY_RUN: + pe_rows = frappe.db.sql("SELECT name FROM \"tabPayment Entry\" WHERE name ~ '^PE-[0-9]+$'") + pe_map = {} + for r in pe_rows: + old = r[0] + num = int(old.replace("PE-", "")) + new = f"PE-{num:010d}" + if old != new: + pe_map[old] = new + + print(f" Will rename: {len(pe_map)} payments") + for old, new in list(pe_map.items())[:3]: + print(f" {old:20s} → {new}") + + if pe_map: + frappe.db.sql('DROP TABLE IF EXISTS _rename_pe') + frappe.db.sql(""" + CREATE TEMP TABLE _rename_pe ( + old_name VARCHAR(140) PRIMARY KEY, new_name VARCHAR(140) NOT NULL + ) + """) + items = list(pe_map.items()) + for start in range(0, len(items), 1000): + batch = items[start:start + 1000] + frappe.db.sql( + "INSERT INTO _rename_pe (old_name, new_name) VALUES " + + ",".join(["(%s, %s)"] * len(batch)), + [v for pair in batch for v in pair] + ) + frappe.db.commit() + + pe_fks = [ + ("tabPayment Entry Reference", "parent", ""), + ("tabGL Entry", "voucher_no", "AND t.voucher_type = 'Payment Entry'"), + ("tabPayment Ledger Entry", "voucher_no", "AND t.voucher_type = 'Payment Entry'"), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Payment Entry'"), + ("tabVersion", "docname", "AND t.ref_doctype = 'Payment Entry'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Payment Entry'"), + ] + for fk_table, fk_col, extra_where in pe_fks: + t0 = time.time() + try: + frappe.db.sql(f""" + UPDATE "{fk_table}" t SET "{fk_col}" = m.new_name + FROM _rename_pe m WHERE t."{fk_col}" = m.old_name {extra_where} + """) + frappe.db.commit() + print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]") + except Exception as e: + frappe.db.rollback() + err = str(e)[:80] + if "does not exist" not in err: + print(f" FK {fk_table}.{fk_col} ERR: {err}") + + # Two-phase rename + print(" Phase A...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '{old}' THEN '_T_{new}'" for old, new in batch) + old_names = "','".join(old for old, _ in batch) + frappe.db.sql(f'UPDATE "tabPayment Entry" SET name = CASE name {cases} END WHERE name IN (\'{old_names}\')') + if (start + 500) % 10000 < 500: + frappe.db.commit() + frappe.db.commit() + print(f" Phase A [{time.time()-t0:.1f}s]") + + print(" Phase B...") + t0 = time.time() + for start in range(0, len(items), 500): + batch = items[start:start + 500] + cases = " ".join(f"WHEN '_T_{new}' THEN '{new}'" for _, new in batch) + temp_names = "','".join(f"_T_{new}" for _, new in batch) + frappe.db.sql(f'UPDATE "tabPayment Entry" SET name = CASE name {cases} END WHERE name IN (\'{temp_names}\')') + if (start + 500) % 10000 < 500: + frappe.db.commit() + frappe.db.commit() + print(f" Phase B [{time.time()-t0:.1f}s]") + + frappe.db.sql('DROP TABLE IF EXISTS _rename_pe') + frappe.db.commit() + + print(f" ✓ Payment Entry [{time.time()-t_start:.1f}s]") +else: + pe_count = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Entry\" WHERE name ~ '^PE-[0-9]+$'")[0][0] + print(f" [DRY] Would rename ~{pe_count} payments") + + +# ═══════════════════════════════════════════════════════════════ +# 3. ISSUE — has legacy_ticket_id column +# ═══════════════════════════════════════════════════════════════ +rename_doctype( + label="Issue", + table="tabIssue", + prefix="ISS", + legacy_col="legacy_ticket_id", + fk_refs=[ + ("tabDispatch Job", "issue", ""), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Issue'"), + ("tabVersion", "docname", "AND t.ref_doctype = 'Issue'"), + ("tabCommunication", "reference_name", "AND t.reference_doctype = 'Issue'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Issue'"), + ], +) + + +# ═══════════════════════════════════════════════════════════════ +# 4. SERVICE LOCATION — has legacy_delivery_id column +# ═══════════════════════════════════════════════════════════════ +rename_doctype( + label="Service Location", + table="tabService Location", + prefix="LOC", + legacy_col="legacy_delivery_id", + fk_refs=[ + ("tabService Subscription", "service_location", ""), + ("tabSubscription", "service_location", ""), + ("tabService Equipment", "service_location", ""), + ("tabIssue", "service_location", ""), + ("tabDispatch Job", "service_location", ""), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Service Location'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Location'"), + ], +) + + +# ═══════════════════════════════════════════════════════════════ +# 5. SERVICE EQUIPMENT — has legacy_device_id column +# ═══════════════════════════════════════════════════════════════ +rename_doctype( + label="Service Equipment", + table="tabService Equipment", + prefix="EQP", + legacy_col="legacy_device_id", + fk_refs=[ + ("tabComment", "reference_name", "AND t.reference_doctype = 'Service Equipment'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Equipment'"), + ], +) + + +# ═══════════════════════════════════════════════════════════════ +# 6. SERVICE SUBSCRIPTION — has legacy_service_id column +# ═══════════════════════════════════════════════════════════════ +rename_doctype( + label="Service Subscription", + table="tabService Subscription", + prefix="SUB", + legacy_col="legacy_service_id", + fk_refs=[ + ("tabComment", "reference_name", "AND t.reference_doctype = 'Service Subscription'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Subscription'"), + ], +) + + +# ═══════════════════════════════════════════════════════════════ +# 7. SUBSCRIPTION (native) — also has legacy_service_id +# ═══════════════════════════════════════════════════════════════ +rename_doctype( + label="Subscription (native)", + table="tabSubscription", + prefix="ASUB", + legacy_col="legacy_service_id", + fk_refs=[ + ("tabSubscription Plan Detail", "parent", ""), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Subscription'"), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Subscription'"), + ], +) + + +# ═══════════════════════════════════════════════════════════════ +# SET NAMING SERIES COUNTERS +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("NAMING SERIES COUNTERS") +print("=" * 70) + +if not DRY_RUN: + set_series("SINV-", 100000) + set_series("PE-", 100000) + set_series("ISS-", 250000) # max legacy=244240 + set_series("LOC-", 100000) + set_series("EQP-", 100000) + set_series("SUB-", 100000) # max legacy=74184 + set_series("ASUB-", 100000) + + +# ═══════════════════════════════════════════════════════════════ +# VERIFICATION +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("VERIFICATION") +print("=" * 70) + +checks = [ + ("tabSales Invoice", "SINV-"), + ("tabPayment Entry", "PE-"), + ("tabIssue", "ISS-"), + ("tabService Location", "LOC-"), + ("tabService Equipment", "EQP-"), + ("tabService Subscription", "SUB-"), + ("tabSubscription", "ASUB-"), +] + +for table, prefix in checks: + try: + total = frappe.db.sql(f'SELECT COUNT(*) FROM "{table}"')[0][0] + correct = frappe.db.sql(f""" + SELECT COUNT(*) FROM "{table}" + WHERE name ~ %s + """, (f'^{prefix}[0-9]{{10}}$',))[0][0] + print(f" {table:30s} total={total:>8d} correct_format={correct:>8d} {'✓' if total == correct else f'({total - correct} remaining)'}") + except Exception as e: + frappe.db.rollback() + print(f" {table}: ERR {str(e)[:50]}") + +# Spot checks +print("\n Spot checks:") +for table, field, value in [ + ("tabSales Invoice", "name", None), + ("tabPayment Entry", "name", None), + ("tabIssue", "name", None), +]: + sample = frappe.db.sql(f'SELECT name FROM "{table}" ORDER BY name LIMIT 3') + print(f" {table}: {[s[0] for s in sample]}") + +# FK orphan checks +print("\n FK consistency:") +fk_checks = [ + ("tabSales Invoice Item", "parent", "tabSales Invoice"), + ("tabPayment Entry Reference", "parent", "tabPayment Entry"), + ("tabGL Entry", "voucher_no", None), # multi-type, skip + ("tabSubscription Plan Detail", "parent", "tabSubscription"), +] +for fk_table, fk_col, parent_table in fk_checks: + if not parent_table: + continue + try: + orphans = frappe.db.sql(f""" + SELECT COUNT(*) FROM "{fk_table}" t + WHERE NOT EXISTS (SELECT 1 FROM "{parent_table}" p WHERE p.name = t."{fk_col}") + """)[0][0] + status = "✓" if orphans == 0 else f"WARNING: {orphans} orphaned!" + print(f" {fk_table}.{fk_col} → {parent_table}: {status}") + except Exception as e: + frappe.db.rollback() + +frappe.clear_cache() +print("\n✓ All done — cache cleared") +print(" Next: bench --site erp.gigafibre.ca clear-cache") diff --git a/scripts/migration/rename_customers.py b/scripts/migration/rename_customers.py new file mode 100644 index 0000000..a08e38a --- /dev/null +++ b/scripts/migration/rename_customers.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +""" +Rename all Customer IDs to CUST-{legacy_customer_id}. + +Handles all FK references across the entire ERPNext database: + - Sales Invoice, Payment Entry, GL Entry, PLE, Issues, Locations, + Subscriptions, Equipment, Dispatch Jobs, Dynamic Links, Comments, etc. + +Approach: + 1. Build old_name → new_name mapping from legacy_customer_id + 2. Create temp mapping table in PostgreSQL + 3. Bulk UPDATE each FK table using JOIN to mapping table + 4. Rename Customer records themselves + 5. Set naming series counter for new customers + 6. Verify all references updated + +Run inside erpnext-backend-1: + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/rename_customers.py [--dry-run] + +Estimated time: 5-10 minutes (millions of rows to update) +""" +import sys +import os +import time +from datetime import datetime, timezone + +os.chdir("/home/frappe/frappe-bench/sites") +import frappe + +frappe.init(site="erp.gigafibre.ca", sites_path=".") +frappe.connect() +frappe.local.flags.ignore_permissions = True +print(f"Connected: {frappe.local.site}") + +DRY_RUN = "--dry-run" in sys.argv +# Use legacy customer_id as the new name: CUST-{legacy_customer_id} +USE_LEGACY_CODE = "--legacy-code" in sys.argv +# Use zero-padded numeric: CUST-{legacy_account_id:015d} +USE_NUMERIC = "--numeric" in sys.argv + +if not USE_LEGACY_CODE and not USE_NUMERIC: + USE_LEGACY_CODE = True # default + +if DRY_RUN: + print("*** DRY RUN — no changes will be written ***") + +mode_label = "legacy_customer_id" if USE_LEGACY_CODE else "zero-padded numeric" +print(f"Naming mode: {mode_label}") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 1: BUILD MAPPING +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 1: BUILD NAME MAPPING") +print("=" * 70) +t0 = time.time() + +customers = frappe.db.sql(""" + SELECT name, legacy_account_id, legacy_customer_id, customer_name + FROM "tabCustomer" + ORDER BY legacy_account_id +""", as_dict=True) + +mapping = {} # old_name → new_name +collisions = [] +new_names_used = set() + +for c in customers: + old_name = c["name"] + + if USE_LEGACY_CODE: + cid = c.get("legacy_customer_id") or "" + if cid: + new_name = f"CUST-{cid}" + else: + # Fallback to zero-padded account ID + new_name = f"CUST-{c['legacy_account_id']:015d}" + else: + new_name = f"CUST-{c['legacy_account_id']:015d}" + + # Check for collision + if new_name in new_names_used: + collisions.append((old_name, new_name, c["customer_name"])) + # Append account ID to disambiguate + new_name = f"{new_name}-{c['legacy_account_id']}" + + new_names_used.add(new_name) + + if old_name != new_name: + mapping[old_name] = new_name + +# Stats +unchanged = len(customers) - len(mapping) +print(f" Total customers: {len(customers)}") +print(f" Will rename: {len(mapping)}") +print(f" Already correct: {unchanged}") +print(f" Collisions resolved: {len(collisions)}") + +if collisions: + print(f" Collision examples:") + for old, new, name in collisions[:5]: + print(f" {old} → {new} ({name})") + +# Show sample mappings +print(f"\n Sample renames:") +for old, new in list(mapping.items())[:10]: + cust = next((c for c in customers if c["name"] == old), {}) + print(f" {old:25s} → {new:35s} ({cust.get('customer_name', '')})") + +if DRY_RUN: + print(f"\n [{time.time()-t0:.1f}s]") + print("\n*** DRY RUN complete — no changes made ***") + sys.exit(0) + + +# ═══════════════════════════════════════════════════════════════ +# STEP 2: CREATE TEMP MAPPING TABLE +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 2: CREATE TEMP MAPPING TABLE") +print("=" * 70) + +frappe.db.sql('DROP TABLE IF EXISTS _customer_rename_map') +frappe.db.sql(""" + CREATE TEMP TABLE _customer_rename_map ( + old_name VARCHAR(140) PRIMARY KEY, + new_name VARCHAR(140) NOT NULL + ) +""") + +# Insert in batches +batch = [] +for old, new in mapping.items(): + batch.append((old, new)) + if len(batch) >= 1000: + frappe.db.sql( + "INSERT INTO _customer_rename_map (old_name, new_name) VALUES " + + ",".join(["(%s, %s)"] * len(batch)), + [v for pair in batch for v in pair] + ) + batch = [] +if batch: + frappe.db.sql( + "INSERT INTO _customer_rename_map (old_name, new_name) VALUES " + + ",".join(["(%s, %s)"] * len(batch)), + [v for pair in batch for v in pair] + ) + +frappe.db.commit() +print(f" Inserted {len(mapping)} mappings into temp table") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 3: UPDATE ALL FK REFERENCES +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 3: UPDATE FK REFERENCES") +print("=" * 70) + +# All tables with customer references: (table, column, extra_where) +fk_tables = [ + ("tabSales Invoice", "customer", ""), + ("tabPayment Entry", "party", ""), + ("tabGL Entry", "party", ""), + ("tabPayment Ledger Entry", "party", ""), + ("tabIssue", "customer", ""), + ("tabService Location", "customer", ""), + ("tabService Subscription", "customer", ""), + ("tabSubscription", "party", ""), + ("tabService Equipment", "customer", ""), + ("tabDispatch Job", "customer", ""), + ("tabDynamic Link", "link_name", "AND t.link_doctype = 'Customer'"), + ("tabComment", "reference_name", "AND t.reference_doctype = 'Customer'"), + ("tabCommunication", "reference_name", "AND t.reference_doctype = 'Customer'"), + # Version log + ("tabVersion", "docname", "AND t.ref_doctype = 'Customer'"), + # Sales taxes (parent references invoice, not customer directly) + # Payment Entry Reference has no direct customer column +] + +total_updated = 0 +for table, col, extra_where in fk_tables: + t0 = time.time() + try: + result = frappe.db.sql(f""" + UPDATE "{table}" t + SET "{col}" = m.new_name + FROM _customer_rename_map m + WHERE t."{col}" = m.old_name + {extra_where} + """) + # Get affected rows + affected = frappe.db.sql(f""" + SELECT COUNT(*) FROM "{table}" t + INNER JOIN _customer_rename_map m ON t."{col}" = m.old_name + {extra_where.replace('t.', f'"{table}".')} + """) + # Actually just count what we updated - the UPDATE already ran + frappe.db.commit() + elapsed = time.time() - t0 + print(f" {table:35s} {col:20s} updated [{elapsed:.1f}s]") + total_updated += 1 + except Exception as e: + frappe.db.rollback() + err_msg = str(e)[:80] + if "does not exist" in err_msg or "column" in err_msg: + pass # Table/column doesn't exist, skip + else: + print(f" {table:35s} {col:20s} ERR: {err_msg}") + +print(f"\n Updated {total_updated} FK tables") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 4: RENAME CUSTOMER RECORDS THEMSELVES +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 4: RENAME CUSTOMER RECORDS") +print("=" * 70) +t0 = time.time() + +# Must rename customer records — update the `name` column +# Do this AFTER FK updates to avoid FK constraint issues +frappe.db.sql(""" + UPDATE "tabCustomer" t + SET name = m.new_name + FROM _customer_rename_map m + WHERE t.name = m.old_name +""") +frappe.db.commit() +print(f" Renamed {len(mapping)} customers [{time.time()-t0:.1f}s]") + + +# ═══════════════════════════════════════════════════════════════ +# STEP 5: SET NAMING SERIES COUNTER +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 5: SET NAMING SERIES COUNTER") +print("=" * 70) + +# For new customers, ERPNext will use naming_series. +# We need to make sure it doesn't collide. +# Since legacy customer_ids are alphanumeric, numeric auto-increment won't collide. +# But let's set a safe starting point anyway. + +# Check if CUST- series exists +series = frappe.db.sql(""" + SELECT name, current FROM "tabSeries" WHERE name = 'CUST-' +""", as_dict=True) + +if series: + print(f" Current CUST- series: {series[0]['current']}") +else: + print(" No CUST- series found, creating...") + frappe.db.sql(""" + INSERT INTO "tabSeries" (name, current) VALUES ('CUST-', 100000) + ON CONFLICT (name) DO UPDATE SET current = GREATEST("tabSeries".current, 100000) + """) + frappe.db.commit() + print(" Set CUST- counter to 100000") + +# Note: The naming_series format should produce CUST-NNNNN or CUST-.##### +# New customers will get CUST-100001, CUST-100002, etc. + + +# ═══════════════════════════════════════════════════════════════ +# STEP 6: CLEANUP TEMP TABLE +# ═══════════════════════════════════════════════════════════════ +frappe.db.sql('DROP TABLE IF EXISTS _customer_rename_map') +frappe.db.commit() + + +# ═══════════════════════════════════════════════════════════════ +# STEP 7: VERIFY +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("STEP 7: VERIFICATION") +print("=" * 70) + +cust_count = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer"')[0][0] +print(f" Customers: {cust_count}") + +# Check for any remaining hex-pattern IDs +hex_remaining = frappe.db.sql(""" + SELECT COUNT(*) FROM "tabCustomer" + WHERE name ~ '^CUST-[0-9a-f]{8,}' + AND name !~ '^CUST-[0-9]+$' +""")[0][0] +print(f" Hex-pattern IDs remaining: {hex_remaining} (should be 0)") + +# Spot checks +spots = [ + (4, "LPB4"), + (2393, "Vegpro"), + (13814, "Vegpro 114796350603272"), +] +for legacy_id, label in spots: + c = frappe.db.sql(""" + SELECT name, customer_name, legacy_customer_id + FROM "tabCustomer" WHERE legacy_account_id = %s + """, (legacy_id,), as_dict=True) + if c: + print(f" {label}: {c[0]['name']} ({c[0]['customer_name']})") + +# Check FK consistency: any orphaned references? +for table, col in [("tabSales Invoice", "customer"), ("tabSubscription", "party"), ("tabIssue", "customer")]: + orphans = frappe.db.sql(f""" + SELECT COUNT(*) FROM "{table}" t + WHERE t."{col}" IS NOT NULL + AND t."{col}" != '' + AND NOT EXISTS (SELECT 1 FROM "tabCustomer" c WHERE c.name = t."{col}") + """)[0][0] + if orphans > 0: + print(f" WARNING: {orphans} orphaned references in {table}.{col}!") + else: + print(f" {table}.{col}: all references valid ✓") + + +# ═══════════════════════════════════════════════════════════════ +# DONE +# ═══════════════════════════════════════════════════════════════ +print("\n" + "=" * 70) +print("COMPLETE") +print("=" * 70) +print(" Next: bench --site erp.gigafibre.ca clear-cache") + +frappe.clear_cache() +print(" Cache cleared")