gigafibre-fsm/scripts/migration/migrate_all.py

#!/usr/bin/env python3
"""
Unified migration: Legacy PHP/MariaDB → ERPNext.
Direct PostgreSQL inserts — no Frappe ORM, no HTTP API.

Prerequisites:
  - nuke_data.py already ran (clean slate except Users + Items)
  - Items + Item Groups + Subscription Plans already exist
  - Scheduler is PAUSED

Run inside erpnext-backend-1:
  nohup python3 /tmp/migrate_all.py > /tmp/migrate_all.log 2>&1 &
  tail -f /tmp/migrate_all.log

Import order (respects dependencies):
  Phase 1: Customers (active + terminated) + Contacts + Addresses
  Phase 2: Subscription Plans (from legacy products)
  Phase 3: Subscriptions (depends on Customers + Plans)
  Phase 4: Sales Invoices + line items (depends on Customers + Items)
  Phase 5: Payment Entries + references (depends on Customers + Invoices)
  Phase 6: Issues + Communications (depends on Customers)
  Phase 7: Memos as Comments (depends on Customers)
"""
import pymysql
import psycopg2
import uuid
from datetime import datetime, timezone
from html import unescape

# ============================================================
# Config
# ============================================================
LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

ADMIN = "Administrator"
COMPANY = "TARGO"
TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T"

GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual",
             7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"}

RECURRING_CATS = {4, 9, 17, 21, 32, 33}

PRIORITY_MAP = {0: "Urgent", 1: "High", 2: "Medium", 3: "Low"}
STATUS_MAP = {"open": "Open", "pending": "On Hold", "closed": "Closed"}

MODE_MAP = {
    "ppa": "Bank Draft", "paiement direct": "Bank Draft",
    "carte credit": "Credit Card", "cheque": "Cheque",
    "comptant": "Cash", "reversement": "Bank Draft",
    "credit": "Credit Note", "credit targo": "Credit Note",
    "credit facture": "Credit Note",
}

# ============================================================
# Helpers
# ============================================================
def uid(prefix=""):
    return prefix + uuid.uuid4().hex[:10]

def ts():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")

def clean(val):
    if not val:
        return ""
    return unescape(str(val)).strip()

def ts_to_dt(unix_ts):
    if not unix_ts or unix_ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
    except (ValueError, OSError):
        return None

def ts_to_date(unix_ts):
    if not unix_ts or unix_ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d")
    except (ValueError, OSError):
        return None

def log(msg):
    print("[{}] {}".format(datetime.now(timezone.utc).strftime("%H:%M:%S"), msg), flush=True)

def commit_every(pg, counter, interval=500):
    if counter > 0 and counter % interval == 0:
        pg.commit()
    return counter


# ============================================================
# Phase 1: Customers + Contacts + Addresses
# ============================================================
def phase1_customers(mc, pg):
    log("")
    log("=" * 60)
    log("PHASE 1: Customers + Contacts + Addresses")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)

    # ALL accounts (active=1, suspended=2, terminated=3,4,5)
    cur.execute("SELECT * FROM account ORDER BY id")
    accounts = cur.fetchall()
    log("  {} accounts loaded".format(len(accounts)))

    cur.execute("SELECT * FROM delivery ORDER BY account_id")
    deliveries = cur.fetchall()
    log("  {} deliveries loaded".format(len(deliveries)))

    del_by = {}
    for d in deliveries:
        del_by.setdefault(d["account_id"], []).append(d)

    pgc = pg.cursor()
    now = ts()

    # The central mapping: legacy account_id → CUST-xxx
    cust_map = {}  # account_id → cust_id

    c_ok = c_addr = c_contact = c_err = 0

    for i, a in enumerate(accounts):
        aid = a["id"]
        first = clean(a["first_name"])
        last = clean(a["last_name"])
        company = clean(a["company"])

        if company:
            ctype, cname = "Company", company
        else:
            ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid)

        cust_id = uid("CUST-")
        group = GROUP_MAP.get(a.get("group_id"), "Individual")
        lang = "fr" if a.get("language_id") == "francais" else "en"

        # Active=1, everything else = disabled
        disabled = 0 if a.get("status") == 1 else 1

        # Terminate details for departed customers
        details = None
        if a.get("status") in (3, 4, 5):
            parts = []
            if a.get("terminate_reason"):
                parts.append("Raison: {}".format(clean(a["terminate_reason"])))
            if a.get("terminate_cie"):
                parts.append("Parti chez: {}".format(clean(a["terminate_cie"])))
            if a.get("terminate_note"):
                parts.append("Notes: {}".format(clean(a["terminate_note"])[:500]))
            if a.get("terminate_date"):
                parts.append("Date: {}".format(clean(a["terminate_date"])))
            if parts:
                details = "\n".join(parts)

        try:
            pgc.execute("""
                INSERT INTO "tabCustomer" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, customer_name, customer_type, customer_group,
                    territory, default_currency, language, disabled,
                    legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id,
                    customer_pos_id, customer_details
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'CUST-.YYYY.-', %s, %s, %s,
                    'Canada', 'CAD', %s, %s,
                    %s, %s, %s, %s,
                    %s, %s
                )
            """, (cust_id, now, now, ADMIN, ADMIN,
                  cname, ctype, group, lang, disabled,
                  aid, clean(a.get("customer_id")),
                  1 if a.get("ppa") else 0,
                  clean(a.get("stripe_id")) or None,
                  clean(a.get("customer_id")) or None,
                  details))

            cust_map[aid] = cust_id
            c_ok += 1

            # --- Contact ---
            email = clean(a.get("email"))
            tel = clean(a.get("tel_home"))
            cell = clean(a.get("cell"))

            if first or email:
                cont_id = uid("CONT-")
                full = "{} {}".format(first, last).strip()

                pgc.execute("""
                    INSERT INTO "tabContact" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        first_name, last_name, full_name, email_id, phone, mobile_no, status
                    ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open')
                """, (cont_id, now, now, ADMIN, ADMIN,
                      first or cname, last or None, full or cname,
                      email or None, tel or None, cell or None))

                # Dynamic Link: Contact → Customer
                pgc.execute("""
                    INSERT INTO "tabDynamic Link" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        link_doctype, link_name, link_title, parent, parentfield, parenttype
                    ) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact')
                """, (uid("DL-"), now, now, ADMIN, ADMIN, cust_id, cname, cont_id))

                if email:
                    pgc.execute("""
                        INSERT INTO "tabContact Email" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            email_id, is_primary, parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact')
                    """, (uid("CE-"), now, now, ADMIN, ADMIN, email, cont_id))

                pidx = 1
                if tel:
                    pgc.execute("""
                        INSERT INTO "tabContact Phone" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            phone, is_primary_phone, is_primary_mobile_no,
                            parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact')
                    """, (uid("CP-"), now, now, ADMIN, ADMIN, pidx, tel, cont_id))
                    pidx += 1
                if cell:
                    pgc.execute("""
                        INSERT INTO "tabContact Phone" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            phone, is_primary_phone, is_primary_mobile_no,
                            parent, parentfield, parenttype
                        ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact')
                    """, (uid("CP-"), now, now, ADMIN, ADMIN, pidx, cell, cont_id))

                c_contact += 1

            # --- Addresses ---
            for j, d in enumerate(del_by.get(aid, [])):
                addr1 = clean(d.get("address1"))
                city = clean(d.get("city"))
                if not addr1 and not city:
                    continue

                addr_id = uid("ADDR-")
                title = clean(d.get("name")) or cname

                pgc.execute("""
                    INSERT INTO "tabAddress" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        address_title, address_type, address_line1, city, state,
                        pincode, country, is_primary_address, is_shipping_address
                    ) VALUES (%s, %s, %s, %s, %s, 0, 0,
                              %s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1)
                """, (addr_id, now, now, ADMIN, ADMIN,
                      title, addr1 or "N/A", city or "N/A",
                      clean(d.get("state")) or "QC", clean(d.get("zip")),
                      1 if j == 0 else 0))

                pgc.execute("""
                    INSERT INTO "tabDynamic Link" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        link_doctype, link_name, link_title, parent, parentfield, parenttype
                    ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address')
                """, (uid("DL-"), now, now, ADMIN, ADMIN, j+1, cust_id, cname, addr_id))

                c_addr += 1

        except Exception as e:
            c_err += 1
            pg.rollback()
            if c_err <= 20:
                log("  ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100]))
            continue

        if c_ok % 500 == 0:
            pg.commit()
            log("  [{}/{}] cust={} contact={} addr={} err={}".format(
                i+1, len(accounts), c_ok, c_contact, c_addr, c_err))

    pg.commit()
    log("  Customers: {} | Contacts: {} | Addresses: {} | Errors: {}".format(
        c_ok, c_contact, c_addr, c_err))

    return cust_map


# ============================================================
# Phase 2: Subscription Plans
# ============================================================
def phase2_subscription_plans(mc, pg):
    log("")
    log("=" * 60)
    log("PHASE 2: Subscription Plans")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()
    now = ts()

    cur.execute("""
        SELECT p.id, p.sku, p.price, p.category
        FROM product p
        WHERE p.category IN (4,9,17,21,32,33)
        ORDER BY p.id
    """)
    products = cur.fetchall()

    cur.execute("SELECT id, sku FROM product")
    sku_map = {r["id"]: r["sku"] for r in cur.fetchall()}

    pgc.execute('SELECT plan_name FROM "tabSubscription Plan"')
    existing = set(r[0] for r in pgc.fetchall())

    created = 0
    plan_by_product = {}

    for p in products:
        sku = sku_map.get(p["id"], "UNKNOWN")
        plan_name = "PLAN-{}".format(sku)
        plan_by_product[p["id"]] = plan_name

        if plan_name in existing:
            continue

        price = float(p["price"] or 0)
        try:
            pgc.execute("""
                INSERT INTO "tabSubscription Plan" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    plan_name, item, currency, price_determination, cost,
                    billing_interval, billing_interval_count
                ) VALUES (%s, %s, %s, %s, %s, 0, 0,
                          %s, %s, 'CAD', 'Fixed Rate', %s, 'Month', 1)
            """, (uid("SP-"), now, now, ADMIN, ADMIN, plan_name, sku, price))
            existing.add(plan_name)
            created += 1
        except Exception as e:
            pg.rollback()
            log("  ERR plan {} -> {}".format(sku, str(e)[:80]))

    pg.commit()
    log("  {} plans created, {} already existed".format(created, len(existing) - created))

    return plan_by_product, sku_map


# ============================================================
# Phase 3: Subscriptions
# ============================================================
def phase3_subscriptions(mc, pg, cust_map, plan_by_product):
    log("")
    log("=" * 60)
    log("PHASE 3: Subscriptions")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()
    now = ts()

    cur.execute("""
        SELECT s.id as service_id, s.product_id, s.delivery_id, s.device_id,
               s.date_orig, s.date_next_invoice, s.payment_recurrence,
               s.hijack, s.hijack_price, s.hijack_desc,
               s.radius_user, s.radius_pwd, s.date_end_contract,
               d.account_id
        FROM service s
        JOIN delivery d ON s.delivery_id = d.id
        WHERE s.status = 1
        AND s.product_id IN (SELECT id FROM product WHERE category IN (4,9,17,21,32,33))
        ORDER BY s.id
    """)
    services = cur.fetchall()
    log("  {} active services loaded".format(len(services)))

    ok = skip = err = 0

    for i, s in enumerate(services):
        acct_id = s["account_id"]
        cust_id = cust_map.get(acct_id)
        if not cust_id:
            skip += 1
            continue

        plan_name = plan_by_product.get(s["product_id"])
        if not plan_name:
            skip += 1
            continue

        start_date = ts_to_date(s["date_orig"]) or "2020-01-01"
        sub_id = uid("SUB-")
        discount = abs(float(s["hijack_price"])) if s.get("hijack") and s.get("hijack_price") else 0

        try:
            pgc.execute("""
                INSERT INTO "tabSubscription" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    party_type, party, company, status,
                    start_date, generate_invoice_at, days_until_due,
                    follow_calendar_months, generate_new_invoices_past_due_date,
                    submit_invoice, cancel_at_period_end,
                    sales_tax_template, additional_discount_amount,
                    radius_user, radius_pwd, legacy_service_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'Customer', %s, %s, 'Active',
                    %s, 'Beginning of the current subscription period', 30,
                    1, 1, 0, 0,
                    %s, %s,
                    %s, %s, %s
                )
            """, (sub_id, now, now, ADMIN, ADMIN,
                  cust_id, COMPANY,
                  start_date, TAX_TEMPLATE, discount,
                  s.get("radius_user") or None,
                  s.get("radius_pwd") or None,
                  s["service_id"]))

            pgc.execute("""
                INSERT INTO "tabSubscription Plan Detail" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    plan, qty, parent, parentfield, parenttype
                ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'plans', 'Subscription')
            """, (uid("SPD-"), now, now, ADMIN, ADMIN, plan_name, sub_id))

            ok += 1
        except Exception as e:
            err += 1
            pg.rollback()
            if err <= 10:
                log("  ERR svc#{} -> {}".format(s["service_id"], str(e)[:100]))
            continue

        if ok % 1000 == 0:
            pg.commit()
            log("  [{}/{}] ok={} skip={} err={}".format(i+1, len(services), ok, skip, err))

    pg.commit()
    log("  Subscriptions: {} created | {} skipped | {} errors".format(ok, skip, err))


# ============================================================
# Phase 4: Sales Invoices
# ============================================================
def phase4_invoices(mc, pg, cust_map):
    log("")
    log("=" * 60)
    log("PHASE 4: Sales Invoices (24 months)")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()
    now = ts()

    cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400)
    cur.execute("SELECT * FROM invoice WHERE billing_status = 1 AND date_orig >= %s ORDER BY id", (cutoff,))
    invoices = cur.fetchall()
    log("  {} invoices loaded".format(len(invoices)))

    inv_ids = [i["id"] for i in invoices]
    items_by_inv = {}
    chunk = 10000
    for s in range(0, len(inv_ids), chunk):
        batch = inv_ids[s:s+chunk]
        cur.execute("SELECT * FROM invoice_item WHERE invoice_id IN ({})".format(
            ",".join(["%s"]*len(batch))), batch)
        for r in cur.fetchall():
            items_by_inv.setdefault(r["invoice_id"], []).append(r)
    log("  {} line items loaded".format(sum(len(v) for v in items_by_inv.values())))

    # ERPNext lookups
    pgc.execute("SELECT item_code FROM \"tabItem\"")
    valid_items = set(r[0] for r in pgc.fetchall())

    pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,))
    receivable = pgc.fetchone()[0]

    # Build GL account mapping: account_number → ERPNext account name
    pgc.execute("""SELECT account_number, name FROM "tabAccount"
                   WHERE root_type = 'Income' AND company = %s AND is_group = 0 AND account_number != ''""", (COMPANY,))
    gl_by_number = {r[0]: r[1] for r in pgc.fetchall()}

    # Fallback generic income account (if no numbered match found)
    pgc.execute("SELECT name FROM \"tabAccount\" WHERE root_type = 'Income' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,))
    income_acct_default = pgc.fetchone()[0]
    log("  GL accounts mapped: {} numbered + default={}".format(len(gl_by_number), income_acct_default))

    # Build SKU → GL account number from legacy: product.sku → product.category → product_cat.num_compte
    cur.execute("""SELECT p.sku, pc.num_compte
                   FROM product p JOIN product_cat pc ON p.category = pc.id
                   WHERE p.sku IS NOT NULL AND pc.num_compte IS NOT NULL""")
    sku_to_gl = {}
    for r in cur.fetchall():
        acct_num = str(int(r["num_compte"])) if r["num_compte"] else None
        if acct_num and acct_num in gl_by_number:
            sku_to_gl[r["sku"]] = gl_by_number[acct_num]
    log("  SKU→GL mapping: {} SKUs mapped".format(len(sku_to_gl)))

    # invoice mapping for payment references
    inv_map = {}  # legacy_invoice_id → SINV-xxx
    inv_ok = inv_skip = inv_err = item_ok = 0

    for i, inv in enumerate(invoices):
        cust_id = cust_map.get(inv["account_id"])
        if not cust_id:
            inv_skip += 1
            continue

        # Look up customer_name from the Customer we created
        # We stored it, but we need it for display. Use a subquery or cache.
        posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01"
        due_date = ts_to_date(inv.get("due_date")) or posting_date
        total = round(float(inv["total_amt"] or 0), 2)
        sinv_name = uid("SINV-")

        try:
            pgc.execute("""
                INSERT INTO "tabSales Invoice" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, customer, company,
                    posting_date, due_date, currency, conversion_rate,
                    selling_price_list, price_list_currency,
                    base_grand_total, grand_total, base_net_total, net_total,
                    base_total, total,
                    outstanding_amount, base_rounded_total, rounded_total,
                    is_return, is_debit_note, disable_rounded_total,
                    debit_to, party_account_currency,
                    status, legacy_invoice_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'ACC-SINV-.YYYY.-', %s, %s,
                    %s, %s, 'CAD', 1,
                    'Standard Selling', 'CAD',
                    %s, %s, %s, %s,
                    %s, %s,
                    %s, %s, %s,
                    0, 0, 1,
                    %s, 'CAD',
                    'Draft', %s
                )
            """, (sinv_name, now, now, ADMIN, ADMIN,
                  cust_id, COMPANY,
                  posting_date, due_date,
                  total, total, total, total,
                  total, total,
                  total, total, total,
                  receivable, inv["id"]))

            for j, li in enumerate(items_by_inv.get(inv["id"], [])):
                sku = clean(li.get("sku")) or "MISC"
                qty = float(li.get("quantity") or 1)
                rate = float(li.get("unitary_price") or 0)
                amount = round(qty * rate, 2)
                desc = clean(li.get("product_name")) or sku
                item_code = sku if sku in valid_items else None

                # Map to correct GL account via SKU → product_cat → num_compte
                income_acct = sku_to_gl.get(sku, income_acct_default)

                pgc.execute("""
                    INSERT INTO "tabSales Invoice Item" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        item_code, item_name, description, qty, rate, amount,
                        base_rate, base_amount, base_net_rate, base_net_amount,
                        net_rate, net_amount,
                        stock_uom, uom, conversion_factor,
                        income_account, cost_center,
                        parent, parentfield, parenttype
                    ) VALUES (
                        %s, %s, %s, %s, %s, 0, %s,
                        %s, %s, %s, %s, %s, %s,
                        %s, %s, %s, %s,
                        %s, %s,
                        'Nos', 'Nos', 1,
                        %s, 'Main - T',
                        %s, 'items', 'Sales Invoice'
                    )
                """, (uid("SII-"), now, now, ADMIN, ADMIN, j+1,
                      item_code, desc[:140], desc[:140], qty, rate, amount,
                      rate, amount, rate, amount,
                      rate, amount,
                      income_acct, sinv_name))
                item_ok += 1

            inv_map[inv["id"]] = sinv_name
            inv_ok += 1

        except Exception as e:
            inv_err += 1
            pg.rollback()
            if inv_err <= 10:
                log("  ERR inv#{} -> {}".format(inv["id"], str(e)[:100]))
            continue

        if inv_ok % 5000 == 0:
            pg.commit()
            log("  [{}/{}] inv={} items={} skip={} err={}".format(
                i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err))

    pg.commit()
    log("  Invoices: {} | Items: {} | Skip: {} | Err: {}".format(inv_ok, item_ok, inv_skip, inv_err))

    return inv_map


# ============================================================
# Phase 5: Payment Entries
# ============================================================
def phase5_payments(mc, pg, cust_map, inv_map):
    log("")
    log("=" * 60)
    log("PHASE 5: Payment Entries (24 months)")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()
    now = ts()

    cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400)
    cur.execute("""SELECT * FROM payment
                   WHERE date_orig >= %s
                   AND type NOT IN ('credit', 'credit targo', 'credit facture')
                   ORDER BY id""", (cutoff,))
    payments = cur.fetchall()
    log("  {} payments loaded".format(len(payments)))

    pay_ids = [p["id"] for p in payments]
    items_by_pay = {}
    chunk = 10000
    for s in range(0, len(pay_ids), chunk):
        batch = pay_ids[s:s+chunk]
        cur.execute("SELECT * FROM payment_item WHERE payment_id IN ({})".format(
            ",".join(["%s"]*len(batch))), batch)
        for r in cur.fetchall():
            items_by_pay.setdefault(r["payment_id"], []).append(r)
    log("  {} payment-invoice links loaded".format(sum(len(v) for v in items_by_pay.values())))

    # ERPNext accounts
    pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,))
    receivable = pgc.fetchone()[0]
    pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type = 'Bank' AND company = %s AND is_group = 0 LIMIT 1", (COMPANY,))
    bank_row = pgc.fetchone()
    bank_acct = bank_row[0] if bank_row else "Banque - T"

    # Ensure legacy_payment_id column exists
    pgc.execute("""SELECT column_name FROM information_schema.columns
                   WHERE table_name = 'tabPayment Entry' AND column_name = 'legacy_payment_id'""")
    has_legacy_col = pgc.fetchone() is not None
    if not has_legacy_col:
        try:
            pgc.execute('ALTER TABLE "tabPayment Entry" ADD COLUMN legacy_payment_id bigint')
            pg.commit()
            has_legacy_col = True
            log("  Added legacy_payment_id column")
        except:
            pg.rollback()

    pay_ok = pay_skip = pay_err = ref_ok = 0

    for i, p in enumerate(payments):
        cust_id = cust_map.get(p["account_id"])
        if not cust_id:
            pay_skip += 1
            continue

        posting_date = ts_to_date(p["date_orig"]) or "2025-01-01"
        amount = round(abs(float(p["amount"] or 0)), 2)
        if amount <= 0:
            pay_skip += 1
            continue

        mode = MODE_MAP.get(p.get("type", ""), "Bank Draft")
        ref = p.get("reference") or ""
        memo = p.get("memo") or ""
        pe_name = uid("PE-")
        creation_dt = ts_to_dt(p["date_orig"]) or now

        try:
            cols = """name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, payment_type, posting_date, company,
                    mode_of_payment, party_type, party,
                    paid_from, paid_to, paid_amount, received_amount,
                    base_paid_amount, base_received_amount,
                    target_exchange_rate, source_exchange_rate,
                    reference_no, reference_date, remarks, status"""
            vals = [pe_name, creation_dt, creation_dt, ADMIN, ADMIN, 0, 0,
                    'ACC-PAY-.YYYY.-', 'Receive', posting_date, COMPANY,
                    mode, 'Customer', cust_id,
                    bank_acct, receivable, amount, amount,
                    amount, amount,
                    1, 1,
                    ref[:140] if ref else None, posting_date,
                    memo[:140] if memo else None, 'Draft']

            if has_legacy_col:
                cols += ", legacy_payment_id"
                vals.append(p["id"])

            placeholders = ",".join(["%s"] * len(vals))
            pgc.execute('INSERT INTO "tabPayment Entry" ({}) VALUES ({})'.format(cols, placeholders), vals)

            # Payment references to invoices
            for j, pi in enumerate(items_by_pay.get(p["id"], [])):
                sinv_name = inv_map.get(pi.get("invoice_id"))
                if not sinv_name:
                    continue
                alloc = round(abs(float(pi.get("amount") or 0)), 2)
                pgc.execute("""
                    INSERT INTO "tabPayment Entry Reference" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        reference_doctype, reference_name, allocated_amount,
                        total_amount, outstanding_amount, exchange_rate,
                        parent, parentfield, parenttype
                    ) VALUES (%s, %s, %s, %s, %s, 0, %s,
                              'Sales Invoice', %s, %s, %s, %s, 1,
                              %s, 'references', 'Payment Entry')
                """, (uid("PER-"), now, now, ADMIN, ADMIN, j+1,
                      sinv_name, alloc, alloc, alloc, pe_name))
                ref_ok += 1

            pay_ok += 1

        except Exception as e:
            pay_err += 1
            pg.rollback()
            if pay_err <= 10:
                log("  ERR pay#{} -> {}".format(p["id"], str(e)[:100]))
            continue

        if pay_ok % 5000 == 0:
            pg.commit()
            log("  [{}/{}] pay={} refs={} skip={} err={}".format(
                i+1, len(payments), pay_ok, ref_ok, pay_skip, pay_err))

    pg.commit()
    log("  Payments: {} | Refs: {} | Skip: {} | Err: {}".format(pay_ok, ref_ok, pay_skip, pay_err))


# ============================================================
# Phase 6: Issues + Communications
# ============================================================
def phase6_issues(mc, pg, cust_map):
    log("")
    log("=" * 60)
    log("PHASE 6: Issues + Communications")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()
    now = ts()

    cur.execute("SELECT * FROM ticket_dept ORDER BY id")
    depts = cur.fetchall()
    cur.execute("SELECT id, email FROM staff WHERE email IS NOT NULL AND email != ''")
    staff_email = {r["id"]: r["email"] for r in cur.fetchall()}

    cur.execute("SELECT * FROM ticket ORDER BY id")
    tickets = cur.fetchall()
    log("  {} tickets loaded".format(len(tickets)))

    cur.execute("""
        SELECT m.* FROM ticket_msg m
        JOIN ticket t ON m.ticket_id = t.id
        WHERE t.status IN ('open', 'pending')
        ORDER BY m.ticket_id, m.id
    """)
    open_msgs = cur.fetchall()
    log("  {} messages for open/pending".format(len(open_msgs)))

    # ERPNext user lookup
    pgc.execute("SELECT name FROM \"tabUser\" WHERE enabled = 1")
    valid_users = set(r[0] for r in pgc.fetchall())

    # Create Issue Priorities
    for pname in ["Urgent", "High", "Medium", "Low"]:
        try:
            pgc.execute("""
                INSERT INTO "tabIssue Priority" (name, creation, modified, modified_by, owner, docstatus, idx)
                VALUES (%s, %s, %s, %s, %s, 0, 0)
            """, (pname, now, now, ADMIN, ADMIN))
        except:
            pg.rollback()
    pg.commit()

    # Create Issue Types
    dept_map = {}
    for d in depts:
        name = clean(d["name"])
        if not name:
            continue
        dept_map[d["id"]] = name
        try:
            pgc.execute("""
                INSERT INTO "tabIssue Type" (name, creation, modified, modified_by, owner, docstatus, idx)
                VALUES (%s, %s, %s, %s, %s, 0, 0)
            """, (name, now, now, ADMIN, ADMIN))
        except:
            pg.rollback()
    pg.commit()
    log("  {} issue types created".format(len(dept_map)))

    # Build message lookup
    msgs_by_ticket = {}
    for m in open_msgs:
        msgs_by_ticket.setdefault(m["ticket_id"], []).append(m)

    # Staff → User mapping
    staff_to_user = {}
    for sid, email in staff_email.items():
        if email in valid_users:
            staff_to_user[sid] = email

    ticket_to_issue = {}
    i_ok = i_skip = i_err = comm_ok = 0

    for i, t in enumerate(tickets):
        tid = t["id"]
        subject = clean(t.get("subject")) or "Ticket #{}".format(tid)
        status = STATUS_MAP.get(t.get("status", "open"), "Open")
        priority = PRIORITY_MAP.get(t.get("priority", 2), "Medium")
        dept_name = dept_map.get(t.get("dept_id"))
        cust_id = cust_map.get(t.get("account_id"))

        opening_date = ts_to_date(t.get("date_create"))
        opening_time = None
        if t.get("date_create") and t["date_create"] > 0:
            try:
                opening_time = datetime.fromtimestamp(int(t["date_create"]), tz=timezone.utc).strftime("%H:%M:%S")
            except (ValueError, OSError):
                pass

        issue_name = uid("ISS-")

        try:
            pgc.execute("""
                INSERT INTO "tabIssue" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, subject, status, priority, issue_type,
                    customer, company, opening_date, opening_time,
                    legacy_ticket_id, is_incident
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'ISS-.YYYY.-', %s, %s, %s, %s,
                    %s, %s, %s, %s,
                    %s, 0
                )
            """, (issue_name, now, now, ADMIN, ADMIN,
                  subject[:255], status, priority, dept_name,
                  cust_id, COMPANY, opening_date, opening_time,
                  tid))

            ticket_to_issue[tid] = issue_name
            i_ok += 1

            # Communications for open/pending tickets
            for m in msgs_by_ticket.get(tid, []):
                msg_text = m.get("msg") or ""
                if not msg_text.strip():
                    continue
                sender = staff_to_user.get(m.get("staff_id"), ADMIN)
                msg_date = ts_to_dt(m.get("date_orig"))

                pgc.execute("""
                    INSERT INTO "tabCommunication" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        subject, content, communication_type, comment_type,
                        reference_doctype, reference_name,
                        sender, communication_date, sent_or_received
                    ) VALUES (
                        %s, %s, %s, %s, %s, 0, 0,
                        %s, %s, 'Communication', 'Comment',
                        'Issue', %s, %s, %s, 'Sent'
                    )
                """, (uid("COM-"), now, now, ADMIN, ADMIN,
                      subject[:255], msg_text, issue_name,
                      sender, msg_date or now))
                comm_ok += 1

        except Exception as e:
            i_err += 1
            pg.rollback()
            if i_err <= 10:
                log("  ERR ticket#{} -> {}".format(tid, str(e)[:100]))
            continue

        if i_ok % 5000 == 0:
            pg.commit()
            log("  [{}/{}] issues={} comm={} err={}".format(
                i+1, len(tickets), i_ok, comm_ok, i_err))

    pg.commit()

    # Parent/child relationships
    log("  Setting parent/child links...")
    parent_set = 0
    for t in tickets:
        if t.get("parent") and t["parent"] > 0:
            child = ticket_to_issue.get(t["id"])
            parent = ticket_to_issue.get(t["parent"])
            if child and parent:
                pgc.execute('UPDATE "tabIssue" SET parent_incident = %s WHERE name = %s', (parent, child))
                pgc.execute('UPDATE "tabIssue" SET is_incident = 1 WHERE name = %s AND is_incident = 0', (parent,))
                parent_set += 1

    pgc.execute("""
        UPDATE "tabIssue" i SET affected_clients = (
            SELECT COUNT(*) FROM "tabIssue" child WHERE child.parent_incident = i.name
        ) WHERE i.is_incident = 1
    """)
    pg.commit()

    log("  Issues: {} | Comms: {} | Parents: {} | Err: {}".format(i_ok, comm_ok, parent_set, i_err))


# ============================================================
# Phase 7: Memos as Comments
# ============================================================
def phase7_memos(mc, pg, cust_map):
    log("")
    log("=" * 60)
    log("PHASE 7: Account Memos → Comments")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    pgc = pg.cursor()

    cur.execute("SELECT * FROM account_memo ORDER BY id")
    memos = cur.fetchall()
    cur.execute("SELECT id, email FROM staff WHERE email IS NOT NULL AND email != ''")
    staff_email = {r["id"]: r["email"] for r in cur.fetchall()}
    log("  {} memos loaded".format(len(memos)))

    pgc.execute("SELECT name FROM \"tabUser\" WHERE enabled = 1")
    valid_users = set(r[0] for r in pgc.fetchall())

    ok = err = 0
    for m in memos:
        cust_id = cust_map.get(m["account_id"])
        if not cust_id:
            err += 1
            continue

        content = clean(m.get("memo")) or "(empty memo)"
        created = ts_to_dt(m.get("date_orig")) or ts_to_dt(m.get("last_updated")) or ts()

        owner = staff_email.get(m.get("staff_id"), ADMIN)
        if owner not in valid_users:
            owner = ADMIN

        try:
            pgc.execute("""
                INSERT INTO "tabComment" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    comment_type, comment_email, content,
                    reference_doctype, reference_name, published
                ) VALUES (%s, %s, %s, %s, %s, 0, 0,
                          'Comment', %s, %s, 'Customer', %s, 0)
            """, (uid("MEMO-"), created, created, owner, owner,
                  owner, content, cust_id))
            ok += 1
        except Exception as e:
            err += 1
            pg.rollback()

        if ok % 5000 == 0 and ok > 0:
            pg.commit()
            log("  {} memos imported...".format(ok))

    pg.commit()
    log("  Memos: {} imported | {} errors".format(ok, err))


# ============================================================
# Main
# ============================================================
def main():
    log("=" * 60)
    log("UNIFIED MIGRATION: Legacy → ERPNext")
    log("=" * 60)

    log("Connecting to legacy MariaDB...")
    mc = pymysql.connect(**LEGACY)
    log("Connecting to ERPNext PostgreSQL...")
    pg = psycopg2.connect(**PG)
    pg.autocommit = False

    # Phase 1: Customers (the central mapping)
    cust_map = phase1_customers(mc, pg)
    log("  >>> cust_map has {} entries".format(len(cust_map)))

    # Phase 2: Subscription Plans
    plan_by_product, sku_map = phase2_subscription_plans(mc, pg)

    # Phase 3: Subscriptions (needs cust_map + plans)
    phase3_subscriptions(mc, pg, cust_map, plan_by_product)

    # Phase 4: Invoices (needs cust_map + items)
    inv_map = phase4_invoices(mc, pg, cust_map)
    log("  >>> inv_map has {} entries".format(len(inv_map)))

    # Phase 5: Payments (needs cust_map + inv_map)
    phase5_payments(mc, pg, cust_map, inv_map)

    # Phase 6: Issues (needs cust_map)
    phase6_issues(mc, pg, cust_map)

    # Phase 7: Memos (needs cust_map)
    phase7_memos(mc, pg, cust_map)

    mc.close()
    pg.close()

    log("")
    log("=" * 60)
    log("ALL PHASES COMPLETE")
    log("Next: bench --site erp.gigafibre.ca clear-cache")
    log("=" * 60)

if __name__ == "__main__":
    main()