gigafibre-fsm/scripts/migration/fix_income_accounts.py

"""
Fix income_account on Sales Invoice Items and GL Entries.
Uses legacy_invoice_id to join with legacy invoice_item → SKU → product_cat → GL account.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/fix_income_accounts.py
"""
import frappe
import pymysql
import os, sys, time

sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
print("Connected:", frappe.local.site)

DEFAULT_INCOME = "Autres produits d'exploitation - T"

# ── Step 1: Build SKU → ERPNext income account ──
print("\n=== Step 1: Build SKU → GL mapping ===")

legacy = pymysql.connect(
    host="legacy-db", user="facturation", password="VD67owoj",
    database="gestionclient", cursorclass=pymysql.cursors.DictCursor
)

with legacy.cursor() as cur:
    cur.execute("""
        SELECT p.sku, pc.num_compte
        FROM product p JOIN product_cat pc ON p.category = pc.id
        WHERE p.sku IS NOT NULL AND p.sku != '' AND pc.num_compte IS NOT NULL
    """)
    sku_to_gl_num = {}
    for r in cur.fetchall():
        if r['num_compte']:
            sku_to_gl_num[r['sku']] = str(int(r['num_compte']))
print(f"  {len(sku_to_gl_num)} SKU → GL number mappings")

# ERPNext account_number → account name
accts = frappe.db.sql("""
    SELECT account_number, name FROM "tabAccount"
    WHERE root_type IN ('Income', 'Expense') AND company = 'TARGO' AND is_group = 0
      AND account_number IS NOT NULL AND account_number != ''
""")
gl_by_number = {r[0]: r[1] for r in accts}
print(f"  {len(gl_by_number)} numbered GL accounts")

sku_to_income = {}
for sku, num in sku_to_gl_num.items():
    if num in gl_by_number:
        sku_to_income[sku] = gl_by_number[num]
print(f"  {len(sku_to_income)} SKUs → named accounts")

# ── Step 2: Load legacy invoice_items grouped by invoice, ordered by id ──
print("\n=== Step 2: Load legacy invoice items with row numbers ===")
t0 = time.time()

with legacy.cursor() as cur:
    cur.execute("""
        SELECT invoice_id, sku,
               ROW_NUMBER() OVER (PARTITION BY invoice_id ORDER BY id) as rn
        FROM invoice_item
        ORDER BY invoice_id, id
    """)
    legacy_items = cur.fetchall()
legacy.close()
print(f"  {len(legacy_items)} legacy items [{time.time()-t0:.0f}s]")

# Build: { legacy_invoice_id → { idx → income_account } }
inv_item_map = {}
mapped = unmapped = 0
for li in legacy_items:
    sku = li['sku'] or ''
    acct = sku_to_income.get(sku)
    if acct:
        inv_id = li['invoice_id']
        idx = li['rn']
        if inv_id not in inv_item_map:
            inv_item_map[inv_id] = {}
        inv_item_map[inv_id][idx] = acct
        mapped += 1
    else:
        unmapped += 1

print(f"  {mapped} items mapped, {unmapped} unmapped")
print(f"  {len(inv_item_map)} invoices with mappable items")

# ── Step 3: Build ERPNext item name → (legacy_invoice_id, idx) lookup ──
print("\n=== Step 3: Update Sales Invoice Items via legacy_invoice_id join ===")
t0 = time.time()

# Process in chunks using legacy_invoice_id from tabSales Invoice
chunk_size = 5000
total_updated = 0
acct_totals = {}

# Get all invoices with legacy_invoice_id
inv_rows = frappe.db.sql("""
    SELECT name, legacy_invoice_id FROM "tabSales Invoice"
    WHERE legacy_invoice_id > 0
    ORDER BY legacy_invoice_id
""")
print(f"  {len(inv_rows)} invoices to process")

for ci in range(0, len(inv_rows), chunk_size):
    chunk = inv_rows[ci:ci+chunk_size]
    inv_names = [r[0] for r in chunk]
    inv_id_map = {r[0]: int(r[1]) for r in chunk}  # erpnext_name → legacy_id

    # Get all items for these invoices
    placeholders = ','.join(['%s'] * len(inv_names))
    items = frappe.db.sql(
        f'SELECT name, parent, idx FROM "tabSales Invoice Item" WHERE parent IN ({placeholders}) AND income_account = %s ORDER BY parent, idx',
        inv_names + [DEFAULT_INCOME]
    )

    updates = []
    for item_name, parent, idx in items:
        legacy_id = inv_id_map.get(parent)
        if legacy_id and legacy_id in inv_item_map and idx in inv_item_map[legacy_id]:
            new_acct = inv_item_map[legacy_id][idx]
            updates.append((new_acct, item_name))
            acct_totals[new_acct] = acct_totals.get(new_acct, 0) + 1

    # Execute updates in batches
    for i in range(0, len(updates), 500):
        batch = updates[i:i+500]
        for new_acct, iname in batch:
            frappe.db.sql(
                'UPDATE "tabSales Invoice Item" SET income_account = %s WHERE name = %s',
                (new_acct, iname)
            )
        frappe.db.commit()

    total_updated += len(updates)
    if (ci // chunk_size) % 10 == 0 or ci + chunk_size >= len(inv_rows):
        elapsed = time.time() - t0
        print(f"  [{ci+len(chunk)}/{len(inv_rows)}] updated={total_updated} [{elapsed:.0f}s]")

print(f"\n  Total items updated: {total_updated}")
print("  By account:")
for acct, cnt in sorted(acct_totals.items(), key=lambda x: -x[1]):
    print(f"    {cnt:>10}  {acct}")

remaining = frappe.db.sql(
    'SELECT COUNT(*) FROM "tabSales Invoice Item" WHERE income_account = %s',
    (DEFAULT_INCOME,)
)[0][0]
total = frappe.db.sql('SELECT COUNT(*) FROM "tabSales Invoice Item"')[0][0]
print(f"\n  Remaining on default: {remaining} / {total}")
print(f"  Time: {time.time()-t0:.0f}s")

# ── Step 4: Update GL Entries based on updated invoice items ──
print("\n=== Step 4: Update GL Entries ===")
t0 = time.time()

# Get aggregated income per invoice per account (non-default only)
inv_accounts = frappe.db.sql("""
    SELECT parent, income_account, SUM(base_net_amount) as total
    FROM "tabSales Invoice Item"
    WHERE income_account != %s
    GROUP BY parent, income_account
""", (DEFAULT_INCOME,))
print(f"  {len(inv_accounts)} (invoice, account) pairs")

# Group by invoice
inv_acct_map = {}
for r in inv_accounts:
    inv_name = r[0]
    if inv_name not in inv_acct_map:
        inv_acct_map[inv_name] = []
    inv_acct_map[inv_name].append({'account': r[1], 'total': float(r[2])})

# For simplicity: assign the GL entry to the dominant (highest total) income account
# A proper fix would split the GL entry, but that requires complex double-entry bookkeeping
gl_updates = {}
for inv_name, accts_list in inv_acct_map.items():
    best = max(accts_list, key=lambda a: a['total'])
    acct = best['account']
    if acct not in gl_updates:
        gl_updates[acct] = []
    gl_updates[acct].append(inv_name)

gl_total = 0
for acct_name, inv_names in gl_updates.items():
    for i in range(0, len(inv_names), 500):
        batch = inv_names[i:i+500]
        placeholders = ','.join(['%s'] * len(batch))
        frappe.db.sql(
            f"""UPDATE "tabGL Entry"
                SET account = %s
                WHERE voucher_type = 'Sales Invoice'
                  AND voucher_no IN ({placeholders})
                  AND account = %s""",
            [acct_name] + batch + [DEFAULT_INCOME]
        )
        frappe.db.commit()
    gl_total += len(inv_names)
    print(f"    {acct_name}: {len(inv_names)} invoices")

print(f"  Total: {gl_total} invoices GL updated [{time.time()-t0:.0f}s]")

# ── Step 5: Verify ──
print("\n=== Step 5: Verify ===")
r = frappe.db.sql("""
    SELECT account, COUNT(*) as cnt, ROUND(SUM(credit)::numeric, 2) as total_credit
    FROM "tabGL Entry"
    WHERE voucher_type = 'Sales Invoice'
      AND credit > 0
    GROUP BY account
    ORDER BY cnt DESC
    LIMIT 25
""")
print("  GL credit entries by account (Sales Invoice):")
for row in r:
    print(f"    {row[1]:>10}  ${row[2]:>14}  {row[0]}")

frappe.destroy()
print("\nDone!")