gigafibre-fsm/scripts/migration/migrate_direct.py

#!/usr/bin/env python3
"""
Direct PostgreSQL migration: Legacy account → ERPNext Customer + Contact + Address.
Writes directly to tabCustomer, tabContact, tabAddress, tabDynamic Link, etc.
No Frappe ORM, no HTTP API — fastest possible method.

Run inside erpnext-backend-1:
  python3 /tmp/migrate_direct.py

Idempotent: skips customers where legacy_account_id already exists.
"""
import pymysql
import psycopg2
import uuid
from html import unescape
from datetime import datetime

# Legacy MariaDB
LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 300}

# ERPNext PostgreSQL
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual",
             7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"}

ADMIN = "Administrator"

def clean(val):
    if not val:
        return ""
    return unescape(str(val)).strip()

def uid(prefix=""):
    return prefix + uuid.uuid4().hex[:10]

def now():
    return datetime.utcnow()

def main():
    print("=== Direct PostgreSQL Migration v5 ===")

    # 1. Read all legacy data in one shot
    print("Reading legacy MariaDB...", flush=True)
    mc = pymysql.connect(**LEGACY)
    cur = mc.cursor(pymysql.cursors.DictCursor)
    cur.execute("SELECT * FROM account WHERE status IN (1,2) ORDER BY id")
    accounts = cur.fetchall()
    cur.execute("SELECT * FROM delivery ORDER BY account_id")
    all_del = cur.fetchall()
    mc.close()
    print("  {} accounts, {} deliveries loaded. MariaDB closed.".format(len(accounts), len(all_del)), flush=True)

    # Delivery lookup
    active_ids = set(a["id"] for a in accounts)
    del_by = {}
    for d in all_del:
        if d["account_id"] in active_ids:
            del_by.setdefault(d["account_id"], []).append(d)

    # 2. Connect ERPNext PostgreSQL
    print("Connecting to ERPNext PostgreSQL...", flush=True)
    pg = psycopg2.connect(**PG)
    pgc = pg.cursor()

    # Existing customers
    pgc.execute('SELECT legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0')
    existing = set(r[0] for r in pgc.fetchall())
    print("  {} already imported, will skip".format(len(existing)), flush=True)

    ts = now()
    c_ok = c_addr = c_contact = c_skip = c_err = 0

    for i, a in enumerate(accounts):
        aid = a["id"]
        if aid in existing:
            c_skip += 1
            continue

        first = clean(a["first_name"])
        last = clean(a["last_name"])
        company = clean(a["company"])
        if company:
            ctype, cname = "Company", company
        else:
            ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid)

        cust_id = uid("CUST-")
        group = GROUP_MAP.get(a["group_id"], "Individual")
        lang = "fr" if a.get("language_id") == "francais" else "en"
        disabled = 0 if a["status"] == 1 else 1

        try:
            # --- Customer ---
            pgc.execute("""
                INSERT INTO "tabCustomer" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, customer_name, customer_type, customer_group,
                    territory, default_currency, language, disabled,
                    legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'CUST-.YYYY.-', %s, %s, %s,
                    'Canada', 'CAD', %s, %s,
                    %s, %s, %s, %s
                )
            """, (cust_id, ts, ts, ADMIN, ADMIN,
                  cname, ctype, group, lang, disabled,
                  aid, clean(a.get("customer_id")),
                  1 if a.get("ppa") else 0,
                  clean(a.get("stripe_id")) or None))
            c_ok += 1

            # --- Contact ---
            email = clean(a.get("email"))
            tel = clean(a.get("tel_home"))
            cell = clean(a.get("cell"))
            if first or email:
                cont_id = uid("CONT-")
                full = "{} {}".format(first, last).strip()
                pgc.execute("""
                    INSERT INTO "tabContact" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        first_name, last_name, full_name, email_id, phone, mobile_no, status
                    ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open')
                """, (cont_id, ts, ts, ADMIN, ADMIN,
                      first or cname, last or None, full or cname,
                      email or None, tel or None, cell or None))

                # Contact → Customer link (Dynamic Link)
                pgc.execute("""
                    INSERT INTO "tabDynamic Link" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        link_doctype, link_name, link_title, parent, parentfield, parenttype
                    ) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact')
                """, (uid("DL-"), ts, ts, ADMIN, ADMIN, cust_id, cname, cont_id))

                # Contact Email (child table)
                if email:
                    pgc.execute("""
                        INSERT INTO "tabContact Email" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            email_id, is_primary, parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact')
                    """, (uid("CE-"), ts, ts, ADMIN, ADMIN, email, cont_id))

                # Contact Phone (child table)
                pidx = 1
                if tel:
                    pgc.execute("""
                        INSERT INTO "tabContact Phone" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            phone, is_primary_phone, is_primary_mobile_no,
                            parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact')
                    """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, tel, cont_id))
                    pidx += 1
                if cell:
                    pgc.execute("""
                        INSERT INTO "tabContact Phone" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            phone, is_primary_phone, is_primary_mobile_no,
                            parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact')
                    """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, cell, cont_id))

                c_contact += 1

            # --- Addresses ---
            for j, d in enumerate(del_by.get(aid, [])):
                addr1 = clean(d.get("address1"))
                city = clean(d.get("city"))
                if not addr1 and not city:
                    continue

                addr_id = uid("ADDR-")
                title = clean(d.get("name")) or cname

                pgc.execute("""
                    INSERT INTO "tabAddress" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        address_title, address_type, address_line1, city, state,
                        pincode, country, is_primary_address, is_shipping_address
                    ) VALUES (%s, %s, %s, %s, %s, 0, 0,
                              %s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1)
                """, (addr_id, ts, ts, ADMIN, ADMIN,
                      title, addr1 or "N/A", city or "N/A",
                      clean(d.get("state")) or "QC", clean(d.get("zip")),
                      1 if j == 0 else 0))

                # Address → Customer link
                pgc.execute("""
                    INSERT INTO "tabDynamic Link" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        link_doctype, link_name, link_title, parent, parentfield, parenttype
                    ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address')
                """, (uid("DL-"), ts, ts, ADMIN, ADMIN, j+1, cust_id, cname, addr_id))

                c_addr += 1

        except Exception as e:
            c_err += 1
            pg.rollback()
            if c_err <= 20:
                print("  ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100]), flush=True)
            continue

        # Commit every 200
        if c_ok % 200 == 0:
            pg.commit()
            print("  [{}/{}] cust={} addr={} contact={} skip={} err={}".format(
                i+1, len(accounts), c_ok, c_addr, c_contact, c_skip, c_err), flush=True)

    pg.commit()
    pg.close()

    print("", flush=True)
    print("=" * 60, flush=True)
    print("Customers: {} created, {} skipped, {} errors".format(c_ok, c_skip, c_err), flush=True)
    print("Contacts:  {}".format(c_contact), flush=True)
    print("Addresses: {}".format(c_addr), flush=True)
    print("=" * 60, flush=True)
    print("", flush=True)
    print("Next: run 'bench --site erp.gigafibre.ca clear-cache'", flush=True)

if __name__ == "__main__":
    main()