gigafibre-fsm/scripts/migration/fix_income_sql.py

"""
Fix income_account using pure SQL bulk updates via a temp mapping table.
Much faster than row-by-row.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/fix_income_sql.py
"""
import frappe
import pymysql
import os, sys, time

sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
print("Connected:", frappe.local.site)

DEFAULT_INCOME = "Autres produits d'exploitation - T"

# ── Step 1: Build SKU → income account ──
print("\n=== Step 1: Build SKU → GL mapping ===")
legacy = pymysql.connect(
    host="legacy-db", user="facturation", password="VD67owoj",
    database="gestionclient", cursorclass=pymysql.cursors.DictCursor
)
with legacy.cursor() as cur:
    cur.execute("""SELECT p.sku, pc.num_compte FROM product p
                   JOIN product_cat pc ON p.category = pc.id
                   WHERE p.sku IS NOT NULL AND p.sku != '' AND pc.num_compte IS NOT NULL""")
    sku_to_gl_num = {r['sku']: str(int(r['num_compte'])) for r in cur.fetchall() if r['num_compte']}

accts = frappe.db.sql("""SELECT account_number, name FROM "tabAccount"
    WHERE root_type IN ('Income','Expense') AND company='TARGO' AND is_group=0
    AND account_number IS NOT NULL AND account_number != ''""")
gl_by_number = {r[0]: r[1] for r in accts}
sku_to_income = {sku: gl_by_number[num] for sku, num in sku_to_gl_num.items() if num in gl_by_number}
print(f"  {len(sku_to_income)} SKU → account mappings")

# ── Step 2: Load legacy items and build (invoice_id, row_num) → account ──
print("\n=== Step 2: Load legacy items ===")
t0 = time.time()
with legacy.cursor() as cur:
    cur.execute("""
        SELECT invoice_id, sku,
               ROW_NUMBER() OVER (PARTITION BY invoice_id ORDER BY id) as rn
        FROM invoice_item
        ORDER BY invoice_id, id
    """)
    legacy_items = cur.fetchall()
legacy.close()
print(f"  {len(legacy_items)} legacy items [{time.time()-t0:.0f}s]")

# ── Step 3: Create temp mapping table in PostgreSQL ──
print("\n=== Step 3: Create temp mapping table ===")
t0 = time.time()

# Drop if exists
frappe.db.sql("DROP TABLE IF EXISTS _tmp_sku_income_map")
frappe.db.commit()

# Create temp table: (legacy_invoice_id, idx, income_account)
frappe.db.sql("""
    CREATE TABLE _tmp_sku_income_map (
        legacy_invoice_id bigint,
        idx integer,
        income_account varchar(140),
        PRIMARY KEY (legacy_invoice_id, idx)
    )
""")
frappe.db.commit()

# Insert mappings in batches
batch = []
batch_size = 5000
inserted = 0

for li in legacy_items:
    sku = li['sku'] or ''
    acct = sku_to_income.get(sku)
    if acct:
        batch.append((li['invoice_id'], li['rn'], acct))
        if len(batch) >= batch_size:
            args = ','.join(['(%s,%s,%s)'] * len(batch))
            flat = [v for t in batch for v in t]
            frappe.db.sql(f"INSERT INTO _tmp_sku_income_map VALUES {args}", flat)
            frappe.db.commit()
            inserted += len(batch)
            batch = []

if batch:
    args = ','.join(['(%s,%s,%s)'] * len(batch))
    flat = [v for t in batch for v in t]
    frappe.db.sql(f"INSERT INTO _tmp_sku_income_map VALUES {args}", flat)
    frappe.db.commit()
    inserted += len(batch)

print(f"  Inserted {inserted} mappings [{time.time()-t0:.0f}s]")

# Add index
frappe.db.sql("CREATE INDEX idx_tmp_map_lid ON _tmp_sku_income_map (legacy_invoice_id)")
frappe.db.commit()

# ── Step 4: Bulk UPDATE Sales Invoice Items via JOIN ──
print("\n=== Step 4: Bulk update Sales Invoice Items ===")
t0 = time.time()

result = frappe.db.sql("""
    UPDATE "tabSales Invoice Item" sii
    SET income_account = m.income_account
    FROM "tabSales Invoice" si, _tmp_sku_income_map m
    WHERE si.name = sii.parent
      AND m.legacy_invoice_id = si.legacy_invoice_id
      AND m.idx = sii.idx
      AND sii.income_account = %s
""", (DEFAULT_INCOME,))
frappe.db.commit()
print(f"  Update done [{time.time()-t0:.0f}s]")

# Check results
dist = frappe.db.sql("""SELECT income_account, COUNT(*) FROM "tabSales Invoice Item"
    GROUP BY income_account ORDER BY COUNT(*) DESC""")
print("  Income account distribution:")
for row in dist:
    print(f"    {row[1]:>10}  {row[0]}")

# ── Step 5: Update GL Entries ──
print("\n=== Step 5: Update GL Entries ===")
t0 = time.time()

# For each invoice, get the dominant income account from its items
# and update the GL entry
print("  Getting dominant account per invoice...")
dom = frappe.db.sql("""
    SELECT parent, income_account, SUM(ABS(base_net_amount)) as total
    FROM "tabSales Invoice Item"
    WHERE income_account != %s
    GROUP BY parent, income_account
""", (DEFAULT_INCOME,))

# Find dominant per invoice
inv_best = {}
for inv_name, acct, total in dom:
    total = float(total or 0)
    if inv_name not in inv_best or total > inv_best[inv_name][1]:
        inv_best[inv_name] = (acct, total)

print(f"  {len(inv_best)} invoices with mapped accounts")

# Batch update GL entries
by_acct = {}
for inv_name, (acct, _) in inv_best.items():
    by_acct.setdefault(acct, []).append(inv_name)

gl_updated = 0
for acct_name, inv_names in by_acct.items():
    for i in range(0, len(inv_names), 1000):
        batch = inv_names[i:i+1000]
        placeholders = ','.join(['%s'] * len(batch))
        frappe.db.sql(
            f"""UPDATE "tabGL Entry" SET account = %s
                WHERE voucher_type = 'Sales Invoice'
                AND voucher_no IN ({placeholders})
                AND account = %s""",
            [acct_name] + batch + [DEFAULT_INCOME]
        )
        frappe.db.commit()
    gl_updated += len(inv_names)
    print(f"    {acct_name}: {len(inv_names)} invoices")

print(f"  Total: {gl_updated} invoices [{time.time()-t0:.0f}s]")

# ── Step 6: Cleanup and verify ──
print("\n=== Step 6: Verify ===")
frappe.db.sql("DROP TABLE IF EXISTS _tmp_sku_income_map")
frappe.db.commit()

r = frappe.db.sql("""
    SELECT account, COUNT(*) as cnt, ROUND(SUM(credit)::numeric, 2) as total_credit
    FROM "tabGL Entry"
    WHERE voucher_type = 'Sales Invoice' AND credit > 0
    GROUP BY account ORDER BY cnt DESC LIMIT 25
""")
print("  GL credit entries by account (Sales Invoice):")
for row in r:
    print(f"    {row[1]:>10}  ${row[2]:>14}  {row[0]}")

frappe.destroy()
print("\nDone!")