gigafibre-fsm/scripts/migration/import_invoices.py

#!/usr/bin/env python3
"""Import legacy invoices (24 months) as Sales Invoice drafts. Direct PG."""
import pymysql
import psycopg2
import uuid
from datetime import datetime, timezone
from html import unescape

LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

ADMIN = "Administrator"
COMPANY = "TARGO"

def uid(p=""):
    return p + uuid.uuid4().hex[:10]

def now():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")

def ts_to_date(t):
    if not t or t <= 0: return None
    try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d")
    except: return None

def clean(v):
    if not v: return ""
    return unescape(str(v)).strip()

def log(msg):
    print(msg, flush=True)

def main():
    ts = now()
    log("=== Import Invoices (24 months) ===")

    mc = pymysql.connect(**LEGACY)
    cur = mc.cursor(pymysql.cursors.DictCursor)
    cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400)

    cur.execute("""SELECT * FROM invoice WHERE billing_status = 1 AND date_orig >= %s ORDER BY id""", (cutoff,))
    invoices = cur.fetchall()
    log("  {} invoices".format(len(invoices)))

    inv_ids = [i["id"] for i in invoices]
    items_by_inv = {}
    chunk = 10000
    for s in range(0, len(inv_ids), chunk):
        batch = inv_ids[s:s+chunk]
        # Join service to carry delivery_id on each invoice_item row
        cur.execute(
            "SELECT ii.*, s.delivery_id AS service_delivery_id "
            "FROM invoice_item ii "
            "LEFT JOIN service s ON s.id = ii.service_id "
            "WHERE ii.invoice_id IN ({}) "
            "ORDER BY ii.invoice_id, ii.id".format(",".join(["%s"]*len(batch))),
            batch,
        )
        for r in cur.fetchall():
            items_by_inv.setdefault(r["invoice_id"], []).append(r)
    mc.close()
    log("  {} items loaded".format(sum(len(v) for v in items_by_inv.values())))

    pg = psycopg2.connect(**PG)
    pgc = pg.cursor()

    pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0')
    cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()}

    pgc.execute('SELECT item_code FROM "tabItem"')
    valid_items = set(r[0] for r in pgc.fetchall())

    # legacy delivery.id → ERPNext Service Location name
    pgc.execute(
        'SELECT legacy_delivery_id, name FROM "tabService Location" '
        "WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id <> 0"
    )
    loc_by_delivery = dict(pgc.fetchall())
    log("  {} Service Locations with legacy_delivery_id".format(len(loc_by_delivery)))

    pgc.execute("""SELECT name FROM "tabAccount" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1""", (COMPANY,))
    receivable = pgc.fetchone()[0]

    # Build GL account mapping: account_number → ERPNext account name
    pgc.execute("""SELECT account_number, name FROM "tabAccount"
                   WHERE root_type IN ('Income','Expense') AND company = %s AND is_group = 0 AND account_number != ''""", (COMPANY,))
    gl_by_number = {r[0]: r[1] for r in pgc.fetchall()}

    pgc.execute("""SELECT name FROM "tabAccount" WHERE root_type = 'Income' AND company = %s AND is_group = 0 LIMIT 1""", (COMPANY,))
    income_acct_default = pgc.fetchone()[0]
    log("  GL accounts: {} numbered + default={}".format(len(gl_by_number), income_acct_default))

    # Build SKU → GL account from legacy product → product_cat.num_compte
    mc2 = pymysql.connect(**LEGACY)
    cur2 = mc2.cursor(pymysql.cursors.DictCursor)
    cur2.execute("""SELECT p.sku, pc.num_compte
                    FROM product p JOIN product_cat pc ON p.category = pc.id
                    WHERE p.sku IS NOT NULL AND pc.num_compte IS NOT NULL""")
    sku_to_gl = {}
    for r in cur2.fetchall():
        acct_num = str(int(r["num_compte"])) if r["num_compte"] else None
        if acct_num and acct_num in gl_by_number:
            sku_to_gl[r["sku"]] = gl_by_number[acct_num]
    mc2.close()
    log("  SKU→GL mapping: {} SKUs".format(len(sku_to_gl)))

    pgc.execute('SELECT legacy_invoice_id FROM "tabSales Invoice" WHERE legacy_invoice_id > 0')
    existing = set(r[0] for r in pgc.fetchall())
    log("  {} already exist".format(len(existing)))

    inv_ok = inv_skip = inv_err = item_ok = 0

    for i, inv in enumerate(invoices):
        if inv["id"] in existing:
            inv_skip += 1
            continue

        cust_data = cust_map.get(inv["account_id"])
        if not cust_data:
            inv_err += 1
            continue

        cust_name, cust_display = cust_data
        posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01"
        due_date = ts_to_date(inv["due_date"]) or posting_date
        total = round(float(inv["total_amt"] or 0), 2)
        sinv_name = uid("SINV-")

        try:
            pgc.execute("""
                INSERT INTO "tabSales Invoice" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, customer, customer_name, company,
                    posting_date, due_date, currency, conversion_rate,
                    selling_price_list, price_list_currency,
                    base_grand_total, grand_total, base_net_total, net_total,
                    base_total, total,
                    outstanding_amount, base_rounded_total, rounded_total,
                    is_return, is_debit_note, disable_rounded_total,
                    debit_to, party_account_currency,
                    status, legacy_invoice_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'ACC-SINV-.YYYY.-', %s, %s, %s,
                    %s, %s, 'CAD', 1,
                    'Standard Selling', 'CAD',
                    %s, %s, %s, %s,
                    %s, %s,
                    %s, %s, %s,
                    0, 0, 1,
                    %s, 'CAD',
                    'Draft', %s
                )
            """, (sinv_name, ts, ts, ADMIN, ADMIN,
                  cust_name, cust_display, COMPANY,
                  posting_date, due_date,
                  total, total, total, total,
                  total, total,
                  total, total, total,
                  receivable, inv["id"]))

            for j, li in enumerate(items_by_inv.get(inv["id"], [])):
                sku = clean(li.get("sku")) or "MISC"
                qty = float(li.get("quantity") or 1)
                rate = float(li.get("unitary_price") or 0)
                amount = round(qty * rate, 2)
                desc = clean(li.get("product_name")) or sku
                item_code = sku if sku in valid_items else None

                # Map to correct GL account via SKU → product_cat → num_compte
                income_acct = sku_to_gl.get(sku, income_acct_default)

                # Resolve service_location via legacy delivery_id carried on the row
                service_location = loc_by_delivery.get(li.get("service_delivery_id"))

                pgc.execute("""
                    INSERT INTO "tabSales Invoice Item" (
                        name, creation, modified, modified_by, owner, docstatus, idx,
                        item_code, item_name, description, qty, rate, amount,
                        base_rate, base_amount, base_net_rate, base_net_amount,
                        net_rate, net_amount,
                        stock_uom, uom, conversion_factor,
                        income_account, cost_center, service_location,
                        parent, parentfield, parenttype
                    ) VALUES (
                        %s, %s, %s, %s, %s, 0, %s,
                        %s, %s, %s, %s, %s, %s,
                        %s, %s, %s, %s,
                        %s, %s,
                        'Nos', 'Nos', 1,
                        %s, 'Main - T', %s,
                        %s, 'items', 'Sales Invoice'
                    )
                """, (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1,
                      item_code, desc[:140], desc[:140], qty, rate, amount,
                      rate, amount, rate, amount,
                      rate, amount,
                      income_acct, service_location, sinv_name))
                item_ok += 1

            inv_ok += 1
        except Exception as e:
            inv_err += 1
            pg.rollback()
            if inv_err <= 10:
                log("  ERR inv#{} -> {}".format(inv["id"], str(e)[:100]))
            continue

        if inv_ok % 2000 == 0:
            pg.commit()
            log("  [{}/{}] inv={} items={} skip={} err={}".format(i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err))

    pg.commit()
    pg.close()
    log("")
    log("=" * 60)
    log("Invoices: {} created, {} skipped, {} errors".format(inv_ok, inv_skip, inv_err))
    log("Items: {}".format(item_ok))
    log("=" * 60)

if __name__ == "__main__":
    main()