#!/usr/bin/env python3 """ Direct PostgreSQL migration: Legacy account → ERPNext Customer + Contact + Address. Writes directly to tabCustomer, tabContact, tabAddress, tabDynamic Link, etc. No Frappe ORM, no HTTP API — fastest possible method. Run inside erpnext-backend-1: python3 /tmp/migrate_direct.py Idempotent: skips customers where legacy_account_id already exists. """ import pymysql import psycopg2 import uuid from html import unescape from datetime import datetime # Legacy MariaDB LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj", "database": "gestionclient", "connect_timeout": 30, "read_timeout": 300} # ERPNext PostgreSQL PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123", "dbname": "_eb65bdc0c4b1b2d6"} GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual", 7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"} ADMIN = "Administrator" def clean(val): if not val: return "" return unescape(str(val)).strip() def uid(prefix=""): return prefix + uuid.uuid4().hex[:10] def now(): return datetime.utcnow() def main(): print("=== Direct PostgreSQL Migration v5 ===") # 1. Read all legacy data in one shot print("Reading legacy MariaDB...", flush=True) mc = pymysql.connect(**LEGACY) cur = mc.cursor(pymysql.cursors.DictCursor) cur.execute("SELECT * FROM account WHERE status IN (1,2) ORDER BY id") accounts = cur.fetchall() cur.execute("SELECT * FROM delivery ORDER BY account_id") all_del = cur.fetchall() mc.close() print(" {} accounts, {} deliveries loaded. MariaDB closed.".format(len(accounts), len(all_del)), flush=True) # Delivery lookup active_ids = set(a["id"] for a in accounts) del_by = {} for d in all_del: if d["account_id"] in active_ids: del_by.setdefault(d["account_id"], []).append(d) # 2. Connect ERPNext PostgreSQL print("Connecting to ERPNext PostgreSQL...", flush=True) pg = psycopg2.connect(**PG) pgc = pg.cursor() # Existing customers pgc.execute('SELECT legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0') existing = set(r[0] for r in pgc.fetchall()) print(" {} already imported, will skip".format(len(existing)), flush=True) ts = now() c_ok = c_addr = c_contact = c_skip = c_err = 0 for i, a in enumerate(accounts): aid = a["id"] if aid in existing: c_skip += 1 continue first = clean(a["first_name"]) last = clean(a["last_name"]) company = clean(a["company"]) if company: ctype, cname = "Company", company else: ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid) cust_id = uid("CUST-") group = GROUP_MAP.get(a["group_id"], "Individual") lang = "fr" if a.get("language_id") == "francais" else "en" disabled = 0 if a["status"] == 1 else 1 try: # --- Customer --- pgc.execute(""" INSERT INTO "tabCustomer" ( name, creation, modified, modified_by, owner, docstatus, idx, naming_series, customer_name, customer_type, customer_group, territory, default_currency, language, disabled, legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'CUST-.YYYY.-', %s, %s, %s, 'Canada', 'CAD', %s, %s, %s, %s, %s, %s ) """, (cust_id, ts, ts, ADMIN, ADMIN, cname, ctype, group, lang, disabled, aid, clean(a.get("customer_id")), 1 if a.get("ppa") else 0, clean(a.get("stripe_id")) or None)) c_ok += 1 # --- Contact --- email = clean(a.get("email")) tel = clean(a.get("tel_home")) cell = clean(a.get("cell")) if first or email: cont_id = uid("CONT-") full = "{} {}".format(first, last).strip() pgc.execute(""" INSERT INTO "tabContact" ( name, creation, modified, modified_by, owner, docstatus, idx, first_name, last_name, full_name, email_id, phone, mobile_no, status ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open') """, (cont_id, ts, ts, ADMIN, ADMIN, first or cname, last or None, full or cname, email or None, tel or None, cell or None)) # Contact → Customer link (Dynamic Link) pgc.execute(""" INSERT INTO "tabDynamic Link" ( name, creation, modified, modified_by, owner, docstatus, idx, link_doctype, link_name, link_title, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact') """, (uid("DL-"), ts, ts, ADMIN, ADMIN, cust_id, cname, cont_id)) # Contact Email (child table) if email: pgc.execute(""" INSERT INTO "tabContact Email" ( name, creation, modified, modified_by, owner, docstatus, idx, email_id, is_primary, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact') """, (uid("CE-"), ts, ts, ADMIN, ADMIN, email, cont_id)) # Contact Phone (child table) pidx = 1 if tel: pgc.execute(""" INSERT INTO "tabContact Phone" ( name, creation, modified, modified_by, owner, docstatus, idx, phone, is_primary_phone, is_primary_mobile_no, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact') """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, tel, cont_id)) pidx += 1 if cell: pgc.execute(""" INSERT INTO "tabContact Phone" ( name, creation, modified, modified_by, owner, docstatus, idx, phone, is_primary_phone, is_primary_mobile_no, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact') """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, cell, cont_id)) c_contact += 1 # --- Addresses --- for j, d in enumerate(del_by.get(aid, [])): addr1 = clean(d.get("address1")) city = clean(d.get("city")) if not addr1 and not city: continue addr_id = uid("ADDR-") title = clean(d.get("name")) or cname pgc.execute(""" INSERT INTO "tabAddress" ( name, creation, modified, modified_by, owner, docstatus, idx, address_title, address_type, address_line1, city, state, pincode, country, is_primary_address, is_shipping_address ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1) """, (addr_id, ts, ts, ADMIN, ADMIN, title, addr1 or "N/A", city or "N/A", clean(d.get("state")) or "QC", clean(d.get("zip")), 1 if j == 0 else 0)) # Address → Customer link pgc.execute(""" INSERT INTO "tabDynamic Link" ( name, creation, modified, modified_by, owner, docstatus, idx, link_doctype, link_name, link_title, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address') """, (uid("DL-"), ts, ts, ADMIN, ADMIN, j+1, cust_id, cname, addr_id)) c_addr += 1 except Exception as e: c_err += 1 pg.rollback() if c_err <= 20: print(" ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100]), flush=True) continue # Commit every 200 if c_ok % 200 == 0: pg.commit() print(" [{}/{}] cust={} addr={} contact={} skip={} err={}".format( i+1, len(accounts), c_ok, c_addr, c_contact, c_skip, c_err), flush=True) pg.commit() pg.close() print("", flush=True) print("=" * 60, flush=True) print("Customers: {} created, {} skipped, {} errors".format(c_ok, c_skip, c_err), flush=True) print("Contacts: {}".format(c_contact), flush=True) print("Addresses: {}".format(c_addr), flush=True) print("=" * 60, flush=True) print("", flush=True) print("Next: run 'bench --site erp.gigafibre.ca clear-cache'", flush=True) if __name__ == "__main__": main()