#!/usr/bin/env python3 """ Unified migration: Legacy PHP/MariaDB → ERPNext. Direct PostgreSQL inserts — no Frappe ORM, no HTTP API. Prerequisites: - nuke_data.py already ran (clean slate except Users + Items) - Items + Item Groups + Subscription Plans already exist - Scheduler is PAUSED Run inside erpnext-backend-1: nohup python3 /tmp/migrate_all.py > /tmp/migrate_all.log 2>&1 & tail -f /tmp/migrate_all.log Import order (respects dependencies): Phase 1: Customers (active + terminated) + Contacts + Addresses Phase 2: Subscription Plans (from legacy products) Phase 3: Subscriptions (depends on Customers + Plans) Phase 4: Sales Invoices + line items (depends on Customers + Items) Phase 5: Payment Entries + references (depends on Customers + Invoices) Phase 6: Issues + Communications (depends on Customers) Phase 7: Memos as Comments (depends on Customers) """ import pymysql import psycopg2 import uuid from datetime import datetime, timezone from html import unescape # ============================================================ # Config # ============================================================ LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj", "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600} PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123", "dbname": "_eb65bdc0c4b1b2d6"} ADMIN = "Administrator" COMPANY = "TARGO" TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T" GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual", 7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"} RECURRING_CATS = {4, 9, 17, 21, 32, 33} PRIORITY_MAP = {0: "Urgent", 1: "High", 2: "Medium", 3: "Low"} STATUS_MAP = {"open": "Open", "pending": "On Hold", "closed": "Closed"} MODE_MAP = { "ppa": "Bank Draft", "paiement direct": "Bank Draft", "carte credit": "Credit Card", "cheque": "Cheque", "comptant": "Cash", "reversement": "Bank Draft", "credit": "Credit Note", "credit targo": "Credit Note", "credit facture": "Credit Note", } # ============================================================ # Helpers # ============================================================ def uid(prefix=""): return prefix + uuid.uuid4().hex[:10] def ts(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") def clean(val): if not val: return "" return unescape(str(val)).strip() def ts_to_dt(unix_ts): if not unix_ts or unix_ts <= 0: return None try: return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") except (ValueError, OSError): return None def ts_to_date(unix_ts): if not unix_ts or unix_ts <= 0: return None try: return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d") except (ValueError, OSError): return None def log(msg): print("[{}] {}".format(datetime.now(timezone.utc).strftime("%H:%M:%S"), msg), flush=True) def commit_every(pg, counter, interval=500): if counter > 0 and counter % interval == 0: pg.commit() return counter # ============================================================ # Phase 1: Customers + Contacts + Addresses # ============================================================ def phase1_customers(mc, pg): log("") log("=" * 60) log("PHASE 1: Customers + Contacts + Addresses") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) # ALL accounts (active=1, suspended=2, terminated=3,4,5) cur.execute("SELECT * FROM account ORDER BY id") accounts = cur.fetchall() log(" {} accounts loaded".format(len(accounts))) cur.execute("SELECT * FROM delivery ORDER BY account_id") deliveries = cur.fetchall() log(" {} deliveries loaded".format(len(deliveries))) del_by = {} for d in deliveries: del_by.setdefault(d["account_id"], []).append(d) pgc = pg.cursor() now = ts() # The central mapping: legacy account_id → CUST-xxx cust_map = {} # account_id → cust_id c_ok = c_addr = c_contact = c_err = 0 for i, a in enumerate(accounts): aid = a["id"] first = clean(a["first_name"]) last = clean(a["last_name"]) company = clean(a["company"]) if company: ctype, cname = "Company", company else: ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid) cust_id = uid("CUST-") group = GROUP_MAP.get(a.get("group_id"), "Individual") lang = "fr" if a.get("language_id") == "francais" else "en" # Active=1, everything else = disabled disabled = 0 if a.get("status") == 1 else 1 # Terminate details for departed customers details = None if a.get("status") in (3, 4, 5): parts = [] if a.get("terminate_reason"): parts.append("Raison: {}".format(clean(a["terminate_reason"]))) if a.get("terminate_cie"): parts.append("Parti chez: {}".format(clean(a["terminate_cie"]))) if a.get("terminate_note"): parts.append("Notes: {}".format(clean(a["terminate_note"])[:500])) if a.get("terminate_date"): parts.append("Date: {}".format(clean(a["terminate_date"]))) if parts: details = "\n".join(parts) try: pgc.execute(""" INSERT INTO "tabCustomer" ( name, creation, modified, modified_by, owner, docstatus, idx, naming_series, customer_name, customer_type, customer_group, territory, default_currency, language, disabled, legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id, customer_pos_id, customer_details ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'CUST-.YYYY.-', %s, %s, %s, 'Canada', 'CAD', %s, %s, %s, %s, %s, %s, %s, %s ) """, (cust_id, now, now, ADMIN, ADMIN, cname, ctype, group, lang, disabled, aid, clean(a.get("customer_id")), 1 if a.get("ppa") else 0, clean(a.get("stripe_id")) or None, clean(a.get("customer_id")) or None, details)) cust_map[aid] = cust_id c_ok += 1 # --- Contact --- email = clean(a.get("email")) tel = clean(a.get("tel_home")) cell = clean(a.get("cell")) if first or email: cont_id = uid("CONT-") full = "{} {}".format(first, last).strip() pgc.execute(""" INSERT INTO "tabContact" ( name, creation, modified, modified_by, owner, docstatus, idx, first_name, last_name, full_name, email_id, phone, mobile_no, status ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open') """, (cont_id, now, now, ADMIN, ADMIN, first or cname, last or None, full or cname, email or None, tel or None, cell or None)) # Dynamic Link: Contact → Customer pgc.execute(""" INSERT INTO "tabDynamic Link" ( name, creation, modified, modified_by, owner, docstatus, idx, link_doctype, link_name, link_title, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact') """, (uid("DL-"), now, now, ADMIN, ADMIN, cust_id, cname, cont_id)) if email: pgc.execute(""" INSERT INTO "tabContact Email" ( name, creation, modified, modified_by, owner, docstatus, idx, email_id, is_primary, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact') """, (uid("CE-"), now, now, ADMIN, ADMIN, email, cont_id)) pidx = 1 if tel: pgc.execute(""" INSERT INTO "tabContact Phone" ( name, creation, modified, modified_by, owner, docstatus, idx, phone, is_primary_phone, is_primary_mobile_no, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact') """, (uid("CP-"), now, now, ADMIN, ADMIN, pidx, tel, cont_id)) pidx += 1 if cell: pgc.execute(""" INSERT INTO "tabContact Phone" ( name, creation, modified, modified_by, owner, docstatus, idx, phone, is_primary_phone, is_primary_mobile_no, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact') """, (uid("CP-"), now, now, ADMIN, ADMIN, pidx, cell, cont_id)) c_contact += 1 # --- Addresses --- for j, d in enumerate(del_by.get(aid, [])): addr1 = clean(d.get("address1")) city = clean(d.get("city")) if not addr1 and not city: continue addr_id = uid("ADDR-") title = clean(d.get("name")) or cname pgc.execute(""" INSERT INTO "tabAddress" ( name, creation, modified, modified_by, owner, docstatus, idx, address_title, address_type, address_line1, city, state, pincode, country, is_primary_address, is_shipping_address ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1) """, (addr_id, now, now, ADMIN, ADMIN, title, addr1 or "N/A", city or "N/A", clean(d.get("state")) or "QC", clean(d.get("zip")), 1 if j == 0 else 0)) pgc.execute(""" INSERT INTO "tabDynamic Link" ( name, creation, modified, modified_by, owner, docstatus, idx, link_doctype, link_name, link_title, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address') """, (uid("DL-"), now, now, ADMIN, ADMIN, j+1, cust_id, cname, addr_id)) c_addr += 1 except Exception as e: c_err += 1 pg.rollback() if c_err <= 20: log(" ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100])) continue if c_ok % 500 == 0: pg.commit() log(" [{}/{}] cust={} contact={} addr={} err={}".format( i+1, len(accounts), c_ok, c_contact, c_addr, c_err)) pg.commit() log(" Customers: {} | Contacts: {} | Addresses: {} | Errors: {}".format( c_ok, c_contact, c_addr, c_err)) return cust_map # ============================================================ # Phase 2: Subscription Plans # ============================================================ def phase2_subscription_plans(mc, pg): log("") log("=" * 60) log("PHASE 2: Subscription Plans") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() now = ts() cur.execute(""" SELECT p.id, p.sku, p.price, p.category FROM product p WHERE p.category IN (4,9,17,21,32,33) ORDER BY p.id """) products = cur.fetchall() cur.execute("SELECT id, sku FROM product") sku_map = {r["id"]: r["sku"] for r in cur.fetchall()} pgc.execute('SELECT plan_name FROM "tabSubscription Plan"') existing = set(r[0] for r in pgc.fetchall()) created = 0 plan_by_product = {} for p in products: sku = sku_map.get(p["id"], "UNKNOWN") plan_name = "PLAN-{}".format(sku) plan_by_product[p["id"]] = plan_name if plan_name in existing: continue price = float(p["price"] or 0) try: pgc.execute(""" INSERT INTO "tabSubscription Plan" ( name, creation, modified, modified_by, owner, docstatus, idx, plan_name, item, currency, price_determination, cost, billing_interval, billing_interval_count ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, 'CAD', 'Fixed Rate', %s, 'Month', 1) """, (uid("SP-"), now, now, ADMIN, ADMIN, plan_name, sku, price)) existing.add(plan_name) created += 1 except Exception as e: pg.rollback() log(" ERR plan {} -> {}".format(sku, str(e)[:80])) pg.commit() log(" {} plans created, {} already existed".format(created, len(existing) - created)) return plan_by_product, sku_map # ============================================================ # Phase 3: Subscriptions # ============================================================ def phase3_subscriptions(mc, pg, cust_map, plan_by_product): log("") log("=" * 60) log("PHASE 3: Subscriptions") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() now = ts() cur.execute(""" SELECT s.id as service_id, s.product_id, s.delivery_id, s.device_id, s.date_orig, s.date_next_invoice, s.payment_recurrence, s.hijack, s.hijack_price, s.hijack_desc, s.radius_user, s.radius_pwd, s.date_end_contract, d.account_id FROM service s JOIN delivery d ON s.delivery_id = d.id WHERE s.status = 1 AND s.product_id IN (SELECT id FROM product WHERE category IN (4,9,17,21,32,33)) ORDER BY s.id """) services = cur.fetchall() log(" {} active services loaded".format(len(services))) ok = skip = err = 0 for i, s in enumerate(services): acct_id = s["account_id"] cust_id = cust_map.get(acct_id) if not cust_id: skip += 1 continue plan_name = plan_by_product.get(s["product_id"]) if not plan_name: skip += 1 continue start_date = ts_to_date(s["date_orig"]) or "2020-01-01" sub_id = uid("SUB-") discount = abs(float(s["hijack_price"])) if s.get("hijack") and s.get("hijack_price") else 0 try: pgc.execute(""" INSERT INTO "tabSubscription" ( name, creation, modified, modified_by, owner, docstatus, idx, party_type, party, company, status, start_date, generate_invoice_at, days_until_due, follow_calendar_months, generate_new_invoices_past_due_date, submit_invoice, cancel_at_period_end, sales_tax_template, additional_discount_amount, radius_user, radius_pwd, legacy_service_id ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'Customer', %s, %s, 'Active', %s, 'Beginning of the current subscription period', 30, 1, 1, 0, 0, %s, %s, %s, %s, %s ) """, (sub_id, now, now, ADMIN, ADMIN, cust_id, COMPANY, start_date, TAX_TEMPLATE, discount, s.get("radius_user") or None, s.get("radius_pwd") or None, s["service_id"])) pgc.execute(""" INSERT INTO "tabSubscription Plan Detail" ( name, creation, modified, modified_by, owner, docstatus, idx, plan, qty, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'plans', 'Subscription') """, (uid("SPD-"), now, now, ADMIN, ADMIN, plan_name, sub_id)) ok += 1 except Exception as e: err += 1 pg.rollback() if err <= 10: log(" ERR svc#{} -> {}".format(s["service_id"], str(e)[:100])) continue if ok % 1000 == 0: pg.commit() log(" [{}/{}] ok={} skip={} err={}".format(i+1, len(services), ok, skip, err)) pg.commit() log(" Subscriptions: {} created | {} skipped | {} errors".format(ok, skip, err)) # ============================================================ # Phase 4: Sales Invoices # ============================================================ def phase4_invoices(mc, pg, cust_map): log("") log("=" * 60) log("PHASE 4: Sales Invoices (24 months)") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() now = ts() cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400) cur.execute("SELECT * FROM invoice WHERE billing_status = 1 AND date_orig >= %s ORDER BY id", (cutoff,)) invoices = cur.fetchall() log(" {} invoices loaded".format(len(invoices))) inv_ids = [i["id"] for i in invoices] items_by_inv = {} chunk = 10000 for s in range(0, len(inv_ids), chunk): batch = inv_ids[s:s+chunk] cur.execute("SELECT * FROM invoice_item WHERE invoice_id IN ({})".format( ",".join(["%s"]*len(batch))), batch) for r in cur.fetchall(): items_by_inv.setdefault(r["invoice_id"], []).append(r) log(" {} line items loaded".format(sum(len(v) for v in items_by_inv.values()))) # ERPNext lookups pgc.execute("SELECT item_code FROM \"tabItem\"") valid_items = set(r[0] for r in pgc.fetchall()) pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,)) receivable = pgc.fetchone()[0] # Build GL account mapping: account_number → ERPNext account name pgc.execute("""SELECT account_number, name FROM "tabAccount" WHERE root_type = 'Income' AND company = %s AND is_group = 0 AND account_number != ''""", (COMPANY,)) gl_by_number = {r[0]: r[1] for r in pgc.fetchall()} # Fallback generic income account (if no numbered match found) pgc.execute("SELECT name FROM \"tabAccount\" WHERE root_type = 'Income' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,)) income_acct_default = pgc.fetchone()[0] log(" GL accounts mapped: {} numbered + default={}".format(len(gl_by_number), income_acct_default)) # Build SKU → GL account number from legacy: product.sku → product.category → product_cat.num_compte cur.execute("""SELECT p.sku, pc.num_compte FROM product p JOIN product_cat pc ON p.category = pc.id WHERE p.sku IS NOT NULL AND pc.num_compte IS NOT NULL""") sku_to_gl = {} for r in cur.fetchall(): acct_num = str(int(r["num_compte"])) if r["num_compte"] else None if acct_num and acct_num in gl_by_number: sku_to_gl[r["sku"]] = gl_by_number[acct_num] log(" SKU→GL mapping: {} SKUs mapped".format(len(sku_to_gl))) # invoice mapping for payment references inv_map = {} # legacy_invoice_id → SINV-xxx inv_ok = inv_skip = inv_err = item_ok = 0 for i, inv in enumerate(invoices): cust_id = cust_map.get(inv["account_id"]) if not cust_id: inv_skip += 1 continue # Look up customer_name from the Customer we created # We stored it, but we need it for display. Use a subquery or cache. posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01" due_date = ts_to_date(inv.get("due_date")) or posting_date total = round(float(inv["total_amt"] or 0), 2) sinv_name = uid("SINV-") try: pgc.execute(""" INSERT INTO "tabSales Invoice" ( name, creation, modified, modified_by, owner, docstatus, idx, naming_series, customer, company, posting_date, due_date, currency, conversion_rate, selling_price_list, price_list_currency, base_grand_total, grand_total, base_net_total, net_total, base_total, total, outstanding_amount, base_rounded_total, rounded_total, is_return, is_debit_note, disable_rounded_total, debit_to, party_account_currency, status, legacy_invoice_id ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'ACC-SINV-.YYYY.-', %s, %s, %s, %s, 'CAD', 1, 'Standard Selling', 'CAD', %s, %s, %s, %s, %s, %s, %s, %s, %s, 0, 0, 1, %s, 'CAD', 'Draft', %s ) """, (sinv_name, now, now, ADMIN, ADMIN, cust_id, COMPANY, posting_date, due_date, total, total, total, total, total, total, total, total, total, receivable, inv["id"])) for j, li in enumerate(items_by_inv.get(inv["id"], [])): sku = clean(li.get("sku")) or "MISC" qty = float(li.get("quantity") or 1) rate = float(li.get("unitary_price") or 0) amount = round(qty * rate, 2) desc = clean(li.get("product_name")) or sku item_code = sku if sku in valid_items else None # Map to correct GL account via SKU → product_cat → num_compte income_acct = sku_to_gl.get(sku, income_acct_default) pgc.execute(""" INSERT INTO "tabSales Invoice Item" ( name, creation, modified, modified_by, owner, docstatus, idx, item_code, item_name, description, qty, rate, amount, base_rate, base_amount, base_net_rate, base_net_amount, net_rate, net_amount, stock_uom, uom, conversion_factor, income_account, cost_center, parent, parentfield, parenttype ) VALUES ( %s, %s, %s, %s, %s, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'Nos', 'Nos', 1, %s, 'Main - T', %s, 'items', 'Sales Invoice' ) """, (uid("SII-"), now, now, ADMIN, ADMIN, j+1, item_code, desc[:140], desc[:140], qty, rate, amount, rate, amount, rate, amount, rate, amount, income_acct, sinv_name)) item_ok += 1 inv_map[inv["id"]] = sinv_name inv_ok += 1 except Exception as e: inv_err += 1 pg.rollback() if inv_err <= 10: log(" ERR inv#{} -> {}".format(inv["id"], str(e)[:100])) continue if inv_ok % 5000 == 0: pg.commit() log(" [{}/{}] inv={} items={} skip={} err={}".format( i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err)) pg.commit() log(" Invoices: {} | Items: {} | Skip: {} | Err: {}".format(inv_ok, item_ok, inv_skip, inv_err)) return inv_map # ============================================================ # Phase 5: Payment Entries # ============================================================ def phase5_payments(mc, pg, cust_map, inv_map): log("") log("=" * 60) log("PHASE 5: Payment Entries (24 months)") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() now = ts() cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400) cur.execute("""SELECT * FROM payment WHERE date_orig >= %s AND type NOT IN ('credit', 'credit targo', 'credit facture') ORDER BY id""", (cutoff,)) payments = cur.fetchall() log(" {} payments loaded".format(len(payments))) pay_ids = [p["id"] for p in payments] items_by_pay = {} chunk = 10000 for s in range(0, len(pay_ids), chunk): batch = pay_ids[s:s+chunk] cur.execute("SELECT * FROM payment_item WHERE payment_id IN ({})".format( ",".join(["%s"]*len(batch))), batch) for r in cur.fetchall(): items_by_pay.setdefault(r["payment_id"], []).append(r) log(" {} payment-invoice links loaded".format(sum(len(v) for v in items_by_pay.values()))) # ERPNext accounts pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,)) receivable = pgc.fetchone()[0] pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Bank' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,)) bank_row = pgc.fetchone() bank_acct = bank_row[0] if bank_row else "Banque - T" # Ensure legacy_payment_id column exists pgc.execute("""SELECT column_name FROM information_schema.columns WHERE table_name = 'tabPayment Entry' AND column_name = 'legacy_payment_id'""") has_legacy_col = pgc.fetchone() is not None if not has_legacy_col: try: pgc.execute('ALTER TABLE "tabPayment Entry" ADD COLUMN legacy_payment_id bigint') pg.commit() has_legacy_col = True log(" Added legacy_payment_id column") except: pg.rollback() pay_ok = pay_skip = pay_err = ref_ok = 0 for i, p in enumerate(payments): cust_id = cust_map.get(p["account_id"]) if not cust_id: pay_skip += 1 continue posting_date = ts_to_date(p["date_orig"]) or "2025-01-01" amount = round(abs(float(p["amount"] or 0)), 2) if amount <= 0: pay_skip += 1 continue mode = MODE_MAP.get(p.get("type", ""), "Bank Draft") ref = p.get("reference") or "" memo = p.get("memo") or "" pe_name = uid("PE-") creation_dt = ts_to_dt(p["date_orig"]) or now try: cols = """name, creation, modified, modified_by, owner, docstatus, idx, naming_series, payment_type, posting_date, company, mode_of_payment, party_type, party, paid_from, paid_to, paid_amount, received_amount, base_paid_amount, base_received_amount, target_exchange_rate, source_exchange_rate, reference_no, reference_date, remarks, status""" vals = [pe_name, creation_dt, creation_dt, ADMIN, ADMIN, 0, 0, 'ACC-PAY-.YYYY.-', 'Receive', posting_date, COMPANY, mode, 'Customer', cust_id, bank_acct, receivable, amount, amount, amount, amount, 1, 1, ref[:140] if ref else None, posting_date, memo[:140] if memo else None, 'Draft'] if has_legacy_col: cols += ", legacy_payment_id" vals.append(p["id"]) placeholders = ",".join(["%s"] * len(vals)) pgc.execute('INSERT INTO "tabPayment Entry" ({}) VALUES ({})'.format(cols, placeholders), vals) # Payment references to invoices for j, pi in enumerate(items_by_pay.get(p["id"], [])): sinv_name = inv_map.get(pi.get("invoice_id")) if not sinv_name: continue alloc = round(abs(float(pi.get("amount") or 0)), 2) pgc.execute(""" INSERT INTO "tabPayment Entry Reference" ( name, creation, modified, modified_by, owner, docstatus, idx, reference_doctype, reference_name, allocated_amount, total_amount, outstanding_amount, exchange_rate, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Sales Invoice', %s, %s, %s, %s, 1, %s, 'references', 'Payment Entry') """, (uid("PER-"), now, now, ADMIN, ADMIN, j+1, sinv_name, alloc, alloc, alloc, pe_name)) ref_ok += 1 pay_ok += 1 except Exception as e: pay_err += 1 pg.rollback() if pay_err <= 10: log(" ERR pay#{} -> {}".format(p["id"], str(e)[:100])) continue if pay_ok % 5000 == 0: pg.commit() log(" [{}/{}] pay={} refs={} skip={} err={}".format( i+1, len(payments), pay_ok, ref_ok, pay_skip, pay_err)) pg.commit() log(" Payments: {} | Refs: {} | Skip: {} | Err: {}".format(pay_ok, ref_ok, pay_skip, pay_err)) # ============================================================ # Phase 6: Issues + Communications # ============================================================ def phase6_issues(mc, pg, cust_map): log("") log("=" * 60) log("PHASE 6: Issues + Communications") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() now = ts() cur.execute("SELECT * FROM ticket_dept ORDER BY id") depts = cur.fetchall() cur.execute("SELECT id, email FROM staff WHERE email IS NOT NULL AND email != ''") staff_email = {r["id"]: r["email"] for r in cur.fetchall()} cur.execute("SELECT * FROM ticket ORDER BY id") tickets = cur.fetchall() log(" {} tickets loaded".format(len(tickets))) cur.execute(""" SELECT m.* FROM ticket_msg m JOIN ticket t ON m.ticket_id = t.id WHERE t.status IN ('open', 'pending') ORDER BY m.ticket_id, m.id """) open_msgs = cur.fetchall() log(" {} messages for open/pending".format(len(open_msgs))) # ERPNext user lookup pgc.execute("SELECT name FROM \"tabUser\" WHERE enabled = 1") valid_users = set(r[0] for r in pgc.fetchall()) # Create Issue Priorities for pname in ["Urgent", "High", "Medium", "Low"]: try: pgc.execute(""" INSERT INTO "tabIssue Priority" (name, creation, modified, modified_by, owner, docstatus, idx) VALUES (%s, %s, %s, %s, %s, 0, 0) """, (pname, now, now, ADMIN, ADMIN)) except: pg.rollback() pg.commit() # Create Issue Types dept_map = {} for d in depts: name = clean(d["name"]) if not name: continue dept_map[d["id"]] = name try: pgc.execute(""" INSERT INTO "tabIssue Type" (name, creation, modified, modified_by, owner, docstatus, idx) VALUES (%s, %s, %s, %s, %s, 0, 0) """, (name, now, now, ADMIN, ADMIN)) except: pg.rollback() pg.commit() log(" {} issue types created".format(len(dept_map))) # Build message lookup msgs_by_ticket = {} for m in open_msgs: msgs_by_ticket.setdefault(m["ticket_id"], []).append(m) # Staff → User mapping staff_to_user = {} for sid, email in staff_email.items(): if email in valid_users: staff_to_user[sid] = email ticket_to_issue = {} i_ok = i_skip = i_err = comm_ok = 0 for i, t in enumerate(tickets): tid = t["id"] subject = clean(t.get("subject")) or "Ticket #{}".format(tid) status = STATUS_MAP.get(t.get("status", "open"), "Open") priority = PRIORITY_MAP.get(t.get("priority", 2), "Medium") dept_name = dept_map.get(t.get("dept_id")) cust_id = cust_map.get(t.get("account_id")) opening_date = ts_to_date(t.get("date_create")) opening_time = None if t.get("date_create") and t["date_create"] > 0: try: opening_time = datetime.fromtimestamp(int(t["date_create"]), tz=timezone.utc).strftime("%H:%M:%S") except (ValueError, OSError): pass issue_name = uid("ISS-") try: pgc.execute(""" INSERT INTO "tabIssue" ( name, creation, modified, modified_by, owner, docstatus, idx, naming_series, subject, status, priority, issue_type, customer, company, opening_date, opening_time, legacy_ticket_id, is_incident ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'ISS-.YYYY.-', %s, %s, %s, %s, %s, %s, %s, %s, %s, 0 ) """, (issue_name, now, now, ADMIN, ADMIN, subject[:255], status, priority, dept_name, cust_id, COMPANY, opening_date, opening_time, tid)) ticket_to_issue[tid] = issue_name i_ok += 1 # Communications for open/pending tickets for m in msgs_by_ticket.get(tid, []): msg_text = m.get("msg") or "" if not msg_text.strip(): continue sender = staff_to_user.get(m.get("staff_id"), ADMIN) msg_date = ts_to_dt(m.get("date_orig")) pgc.execute(""" INSERT INTO "tabCommunication" ( name, creation, modified, modified_by, owner, docstatus, idx, subject, content, communication_type, comment_type, reference_doctype, reference_name, sender, communication_date, sent_or_received ) VALUES ( %s, %s, %s, %s, %s, 0, 0, %s, %s, 'Communication', 'Comment', 'Issue', %s, %s, %s, 'Sent' ) """, (uid("COM-"), now, now, ADMIN, ADMIN, subject[:255], msg_text, issue_name, sender, msg_date or now)) comm_ok += 1 except Exception as e: i_err += 1 pg.rollback() if i_err <= 10: log(" ERR ticket#{} -> {}".format(tid, str(e)[:100])) continue if i_ok % 5000 == 0: pg.commit() log(" [{}/{}] issues={} comm={} err={}".format( i+1, len(tickets), i_ok, comm_ok, i_err)) pg.commit() # Parent/child relationships log(" Setting parent/child links...") parent_set = 0 for t in tickets: if t.get("parent") and t["parent"] > 0: child = ticket_to_issue.get(t["id"]) parent = ticket_to_issue.get(t["parent"]) if child and parent: pgc.execute('UPDATE "tabIssue" SET parent_incident = %s WHERE name = %s', (parent, child)) pgc.execute('UPDATE "tabIssue" SET is_incident = 1 WHERE name = %s AND is_incident = 0', (parent,)) parent_set += 1 pgc.execute(""" UPDATE "tabIssue" i SET affected_clients = ( SELECT COUNT(*) FROM "tabIssue" child WHERE child.parent_incident = i.name ) WHERE i.is_incident = 1 """) pg.commit() log(" Issues: {} | Comms: {} | Parents: {} | Err: {}".format(i_ok, comm_ok, parent_set, i_err)) # ============================================================ # Phase 7: Memos as Comments # ============================================================ def phase7_memos(mc, pg, cust_map): log("") log("=" * 60) log("PHASE 7: Account Memos → Comments") log("=" * 60) cur = mc.cursor(pymysql.cursors.DictCursor) pgc = pg.cursor() cur.execute("SELECT * FROM account_memo ORDER BY id") memos = cur.fetchall() cur.execute("SELECT id, email FROM staff WHERE email IS NOT NULL AND email != ''") staff_email = {r["id"]: r["email"] for r in cur.fetchall()} log(" {} memos loaded".format(len(memos))) pgc.execute("SELECT name FROM \"tabUser\" WHERE enabled = 1") valid_users = set(r[0] for r in pgc.fetchall()) ok = err = 0 for m in memos: cust_id = cust_map.get(m["account_id"]) if not cust_id: err += 1 continue content = clean(m.get("memo")) or "(empty memo)" created = ts_to_dt(m.get("date_orig")) or ts_to_dt(m.get("last_updated")) or ts() owner = staff_email.get(m.get("staff_id"), ADMIN) if owner not in valid_users: owner = ADMIN try: pgc.execute(""" INSERT INTO "tabComment" ( name, creation, modified, modified_by, owner, docstatus, idx, comment_type, comment_email, content, reference_doctype, reference_name, published ) VALUES (%s, %s, %s, %s, %s, 0, 0, 'Comment', %s, %s, 'Customer', %s, 0) """, (uid("MEMO-"), created, created, owner, owner, owner, content, cust_id)) ok += 1 except Exception as e: err += 1 pg.rollback() if ok % 5000 == 0 and ok > 0: pg.commit() log(" {} memos imported...".format(ok)) pg.commit() log(" Memos: {} imported | {} errors".format(ok, err)) # ============================================================ # Main # ============================================================ def main(): log("=" * 60) log("UNIFIED MIGRATION: Legacy → ERPNext") log("=" * 60) log("Connecting to legacy MariaDB...") mc = pymysql.connect(**LEGACY) log("Connecting to ERPNext PostgreSQL...") pg = psycopg2.connect(**PG) pg.autocommit = False # Phase 1: Customers (the central mapping) cust_map = phase1_customers(mc, pg) log(" >>> cust_map has {} entries".format(len(cust_map))) # Phase 2: Subscription Plans plan_by_product, sku_map = phase2_subscription_plans(mc, pg) # Phase 3: Subscriptions (needs cust_map + plans) phase3_subscriptions(mc, pg, cust_map, plan_by_product) # Phase 4: Invoices (needs cust_map + items) inv_map = phase4_invoices(mc, pg, cust_map) log(" >>> inv_map has {} entries".format(len(inv_map))) # Phase 5: Payments (needs cust_map + inv_map) phase5_payments(mc, pg, cust_map, inv_map) # Phase 6: Issues (needs cust_map) phase6_issues(mc, pg, cust_map) # Phase 7: Memos (needs cust_map) phase7_memos(mc, pg, cust_map) mc.close() pg.close() log("") log("=" * 60) log("ALL PHASES COMPLETE") log("Next: bench --site erp.gigafibre.ca clear-cache") log("=" * 60) if __name__ == "__main__": main()