gigafibre-fsm/scripts/migration/fix_and_invoices.py

#!/usr/bin/env python3
"""
1. Fix Subscription.party → Customer.name (CUST-xxx)
2. Import legacy invoices (last 24 months) as Sales Invoice
Direct PG. Detached.
"""
import pymysql
import psycopg2
import uuid
from datetime import datetime, timezone
from html import unescape

LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

ADMIN = "Administrator"
COMPANY = "TARGO"

def uid(p=""):
    return p + uuid.uuid4().hex[:10]

def now():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")

def ts_to_date(ts):
    if not ts or ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(int(ts), tz=timezone.utc).strftime("%Y-%m-%d")
    except:
        return None

def clean(v):
    if not v: return ""
    return unescape(str(v)).strip()

def log(msg):
    print(msg, flush=True)

def main():
    ts = now()

    # =============================
    # PART 1: Fix Subscription.party
    # =============================
    log("=== Part 1: Fix Subscription.party ===")

    pg = psycopg2.connect(**PG)
    pgc = pg.cursor()

    # Fix party field: match customer_name → actual name
    pgc.execute("""
        UPDATE "tabSubscription" s
        SET party = c.name
        FROM "tabCustomer" c
        WHERE s.party = c.customer_name
        AND s.party_type = 'Customer'
        AND s.party NOT LIKE 'CUST-%'
    """)
    fixed_party = pgc.rowcount
    pg.commit()
    log("  Fixed {} Subscription.party → CUST-xxx".format(fixed_party))

    # Also fix any remaining by legacy_service_id mapping
    pgc.execute("""
        SELECT s.name as sub_name, s.legacy_service_id
        FROM "tabSubscription" s
        WHERE s.party NOT LIKE 'CUST-%' AND s.legacy_service_id > 0
    """)
    remaining = pgc.fetchall()

    if remaining:
        mc = pymysql.connect(**LEGACY)
        cur = mc.cursor(pymysql.cursors.DictCursor)
        cur.execute("SELECT s.id, d.account_id FROM service s JOIN delivery d ON s.delivery_id = d.id WHERE s.status = 1")
        svc_acct = {r["id"]: r["account_id"] for r in cur.fetchall()}
        mc.close()

        pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0')
        cust_map = {r[0]: r[1] for r in pgc.fetchall()}

        fixed2 = 0
        for sub_name, svc_id in remaining:
            acct_id = svc_acct.get(svc_id)
            if acct_id:
                cust_name = cust_map.get(acct_id)
                if cust_name:
                    pgc.execute('UPDATE "tabSubscription" SET party = %s WHERE name = %s', (cust_name, sub_name))
                    fixed2 += 1
        pg.commit()
        log("  Fixed {} more via legacy_service_id".format(fixed2))
    else:
        log("  No remaining fixes needed")
        # Still need legacy data for Part 2
        mc = pymysql.connect(**LEGACY)
        cur = mc.cursor(pymysql.cursors.DictCursor)

    # =============================
    # PART 2: Import Invoices
    # =============================
    log("")
    log("=== Part 2: Import Legacy Invoices (24 months) ===")

    mc = pymysql.connect(**LEGACY)
    cur = mc.cursor(pymysql.cursors.DictCursor)

    # Invoices from last 24 months
    cutoff = int((datetime.now(timezone.utc).timestamp())) - (24 * 30 * 86400)
    cur.execute("""
        SELECT i.id, i.account_id, i.date_orig, i.total_amt, i.billed_amt,
               i.billing_status, i.due_date, i.notes
        FROM invoice i
        WHERE i.billing_status = 1 AND i.date_orig >= %s
        ORDER BY i.id
    """, (cutoff,))
    invoices = cur.fetchall()
    log("  {} invoices to import".format(len(invoices)))

    # Invoice items
    inv_ids = [i["id"] for i in invoices]
    items_by_inv = {}
    if inv_ids:
        # Batch query in chunks
        chunk = 10000
        for start in range(0, len(inv_ids), chunk):
            batch = inv_ids[start:start+chunk]
            placeholders = ",".join(["%s"] * len(batch))
            cur.execute("""
                SELECT invoice_id, sku, quantity, unitary_price, product_name, service_id
                FROM invoice_item WHERE invoice_id IN ({})
            """.format(placeholders), batch)
            for r in cur.fetchall():
                items_by_inv.setdefault(r["invoice_id"], []).append(r)
    mc.close()
    log("  {} invoice items loaded".format(sum(len(v) for v in items_by_inv.values())))

    # Customer mapping
    pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0')
    cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()}

    # Check existing invoices
    pgc.execute('SELECT name FROM "tabSales Invoice" WHERE name LIKE %s', ('SINV-LEG-%',))
    existing_inv = set(r[0] for r in pgc.fetchall())

    # Item existence check
    pgc.execute('SELECT item_code FROM "tabItem"')
    valid_items = set(r[0] for r in pgc.fetchall())

    # Get receivable + income accounts
    pgc.execute("""SELECT name FROM "tabAccount" WHERE account_type = 'Receivable' AND company = 'TARGO' AND is_group = 0 LIMIT 1""")
    receivable = pgc.fetchone()[0]
    pgc.execute("""SELECT name FROM "tabAccount" WHERE root_type = 'Income' AND company = 'TARGO' AND is_group = 0 LIMIT 1""")
    income_row = pgc.fetchone()
    income_acct = income_row[0] if income_row else "Revenus autres - T"

    inv_ok = inv_skip = inv_err = item_ok = 0

    for i, inv in enumerate(invoices):
        inv_name = "SINV-LEG-{}".format(inv["id"])
        if inv_name in existing_inv:
            inv_skip += 1
            continue

        cust_data = cust_map.get(inv["account_id"])
        if not cust_data:
            inv_err += 1
            continue

        cust_name, cust_display = cust_data
        posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01"
        due_date = ts_to_date(inv["due_date"]) or posting_date
        total = float(inv["total_amt"] or 0)

        try:
            # Sales Invoice header
            pgc.execute("""
                INSERT INTO "tabSales Invoice" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, title, customer, customer_name, company,
                    posting_date, due_date, currency, conversion_rate,
                    selling_price_list, price_list_currency,
                    base_grand_total, grand_total, base_net_total, net_total,
                    base_total, total,
                    outstanding_amount, base_rounded_total, rounded_total,
                    is_return, is_debit_note, disable_rounded_total,
                    debit_to, party_account_currency,
                    status, docstatus
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'ACC-SINV-.YYYY.-', %s, %s, %s, %s,
                    %s, %s, 'CAD', 1,
                    'Standard Selling', 'CAD',
                    %s, %s, %s, %s,
                    %s, %s,
                    %s, %s, %s,
                    0, 0, 1,
                    %s, 'CAD',
                    'Draft', 0
                )
            """, (inv_name, ts, ts, ADMIN, ADMIN,
                  cust_display, cust_name, cust_display, COMPANY,
                  posting_date, due_date,
                  total, total, total, total,
                  total, total,
                  total, total, total,
                  receivable))

            # Invoice items
            line_items = items_by_inv.get(inv["id"], [])
            for j, li in enumerate(line_items):
                sku = clean(li.get("sku")) or "MISC"
                qty = float(li.get("quantity") or 1)
                rate = float(li.get("unitary_price") or 0)
                amount = round(qty * rate, 2)
                desc = clean(li.get("product_name")) or sku

                # Use valid item or fallback
                item_code = sku if sku in valid_items else None

                pgc.execute("""
                    INSERT INTO "tabSales Invoice Item" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        item_code, item_name, description, qty, rate, amount,
                        base_rate, base_amount, base_net_rate, base_net_amount,
                        net_rate, net_amount,
                        stock_uom, uom, conversion_factor,
                        income_account, cost_center,
                        parent, parentfield, parenttype
                    ) VALUES (
                        %s, %s, %s, %s, %s, 0, %s,
                        %s, %s, %s, %s, %s, %s,
                        %s, %s, %s, %s,
                        %s, %s,
                        'Nos', 'Nos', 1,
                        %s, 'Main - T',
                        %s, 'items', 'Sales Invoice'
                    )
                """, (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1,
                      item_code, desc[:140], desc[:140], qty, rate, amount,
                      rate, amount, rate, amount,
                      rate, amount,
                      income_acct,
                      inv_name))
                item_ok += 1

            inv_ok += 1

        except Exception as e:
            inv_err += 1
            pg.rollback()
            if inv_err <= 10:
                log("  ERR inv#{} -> {}".format(inv["id"], str(e)[:100]))
            continue

        if inv_ok % 2000 == 0:
            pg.commit()
            log("  [{}/{}] inv={} items={} skip={} err={}".format(
                i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err))

    pg.commit()
    pg.close()

    log("")
    log("=" * 60)
    log("Subscriptions fixed: {}".format(fixed_party))
    log("Invoices: {} created, {} skipped, {} errors".format(inv_ok, inv_skip, inv_err))
    log("Invoice Items: {}".format(item_ok))
    log("=" * 60)
    log("bench --site erp.gigafibre.ca clear-cache")

if __name__ == "__main__":
    main()