gigafibre-fsm/scripts/migration/fix_invoice_outstanding.py

"""
Fix outstanding_amount on Sales Invoices using legacy system as source of truth.

Legacy invoice table:
  - billing_status: 1 = paid, 0 = unpaid
  - total_amt: invoice total
  - billed_amt: amount that has been paid
  - montant_du (computed) = total_amt - billed_amt

ERPNext migration created wrong payment allocations, causing phantom outstanding.
This script reads every legacy invoice's real status and corrects ERPNext.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/fix_invoice_outstanding.py
"""
import frappe
import pymysql
import os
import time

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

T_TOTAL = time.time()

# ═══════════════════════════════════════════════════════════════
# PHASE 1: Load legacy invoice statuses
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 1: LOAD LEGACY INVOICE DATA")
print("="*60)

conn = pymysql.connect(
    host="legacy-db",
    user="facturation",
    password="*******",
    database="gestionclient",
    cursorclass=pymysql.cursors.DictCursor
)

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, total_amt, billed_amt, billing_status
        FROM invoice
    """)
    legacy = {}
    for row in cur.fetchall():
        total = float(row["total_amt"] or 0)
        billed = float(row["billed_amt"] or 0)
        montant_du = round(total - billed, 2)
        if montant_du < 0:
            montant_du = 0.0
        legacy[row["id"]] = {
            "montant_du": montant_du,
            "billing_status": row["billing_status"],
            "total_amt": total,
            "billed_amt": billed,
        }

conn.close()
print("Legacy invoices loaded: {:,}".format(len(legacy)))

status_dist = {}
for v in legacy.values():
    s = v["billing_status"]
    status_dist[s] = status_dist.get(s, 0) + 1
print("  billing_status=0 (unpaid): {:,}".format(status_dist.get(0, 0)))
print("  billing_status=1 (paid):   {:,}".format(status_dist.get(1, 0)))

# ═══════════════════════════════════════════════════════════════
# PHASE 2: Load ERPNext invoices and find mismatches
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 2: FIND MISMATCHES")
print("="*60)

erp_invoices = frappe.db.sql("""
    SELECT name, outstanding_amount, status, grand_total
    FROM "tabSales Invoice"
    WHERE docstatus = 1
""", as_dict=True)

print("ERPNext submitted invoices: {:,}".format(len(erp_invoices)))

mismatches = []
matched = 0
no_legacy = 0
errors = []

for inv in erp_invoices:
    # Extract legacy ID from SINV-{id}
    try:
        legacy_id = int(inv["name"].split("-")[1])
    except (IndexError, ValueError):
        errors.append(inv["name"])
        continue

    leg = legacy.get(legacy_id)
    if not leg:
        no_legacy += 1
        continue

    erp_out = round(float(inv["outstanding_amount"] or 0), 2)
    legacy_out = leg["montant_du"]

    if abs(erp_out - legacy_out) > 0.005:
        mismatches.append({
            "name": inv["name"],
            "legacy_id": legacy_id,
            "erp_outstanding": erp_out,
            "legacy_outstanding": legacy_out,
            "erp_status": inv["status"],
            "grand_total": float(inv["grand_total"] or 0),
            "billing_status": leg["billing_status"],
        })
    else:
        matched += 1

print("Correct (matched):    {:,}".format(matched))
print("MISMATCHED:           {:,}".format(len(mismatches)))
print("No legacy record:     {:,}".format(no_legacy))
print("Parse errors:         {:,}".format(len(errors)))

# Breakdown
should_be_zero = [m for m in mismatches if m["legacy_outstanding"] == 0]
should_be_nonzero = [m for m in mismatches if m["legacy_outstanding"] > 0]
print("\n  Legacy says PAID (should be 0):    {:,} invoices".format(len(should_be_zero)))
print("  Legacy says UNPAID (real balance): {:,} invoices".format(len(should_be_nonzero)))

phantom_total = sum(m["erp_outstanding"] for m in should_be_zero)
print("  Phantom outstanding to clear:      ${:,.2f}".format(phantom_total))

# Sample
print("\nSample mismatches:")
for m in mismatches[:10]:
    print("  {} | ERPNext={:.2f} → Legacy={:.2f} | billing_status={}".format(
        m["name"], m["erp_outstanding"], m["legacy_outstanding"], m["billing_status"]))

# ═══════════════════════════════════════════════════════════════
# PHASE 3: Triple-check before fixing
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 3: TRIPLE-CHECK")
print("="*60)

# Verify with specific known-good cases
# Invoice 634020 for our problem customer — legacy says paid, ERPNext says paid
test_634020 = legacy.get(634020)
print("Invoice 634020 (should be paid):")
print("  Legacy: montant_du={}, billing_status={}".format(
    test_634020["montant_du"] if test_634020 else "MISSING",
    test_634020["billing_status"] if test_634020 else "MISSING"))
erp_634020 = frappe.db.sql('SELECT outstanding_amount, status FROM "tabSales Invoice" WHERE name=%s',
                            ("SINV-634020",), as_dict=True)
if erp_634020:
    print("  ERPNext: outstanding={}, status={}".format(erp_634020[0]["outstanding_amount"], erp_634020[0]["status"]))

# Invoice 607832 — legacy says paid (billing_status=1, billed_amt=14.00), ERPNext says Overdue
test_607832 = legacy.get(607832)
print("\nInvoice 607832 (phantom overdue):")
print("  Legacy: montant_du={}, billing_status={}".format(
    test_607832["montant_du"] if test_607832 else "MISSING",
    test_607832["billing_status"] if test_607832 else "MISSING"))
erp_607832 = frappe.db.sql('SELECT outstanding_amount, status FROM "tabSales Invoice" WHERE name=%s',
                            ("SINV-607832",), as_dict=True)
if erp_607832:
    print("  ERPNext: outstanding={}, status={}".format(erp_607832[0]["outstanding_amount"], erp_607832[0]["status"]))
    print("  WILL FIX → outstanding=0, status=Paid")

# Verify an invoice that IS genuinely unpaid in legacy (billing_status=0)
unpaid_sample = [m for m in mismatches if m["billing_status"] == 0][:3]
if unpaid_sample:
    print("\nSample genuinely unpaid invoices:")
    for m in unpaid_sample:
        print("  {} | legacy_outstanding={:.2f} (genuinely owed)".format(m["name"], m["legacy_outstanding"]))

# ═══════════════════════════════════════════════════════════════
# PHASE 4: APPLY FIXES
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 4: APPLY FIXES")
print("="*60)

fixed = 0
batch_size = 2000

for i in range(0, len(mismatches), batch_size):
    batch = mismatches[i:i+batch_size]
    for m in batch:
        new_outstanding = m["legacy_outstanding"]

        # Determine correct status
        if new_outstanding <= 0:
            new_status = "Paid"
        elif new_outstanding >= m["grand_total"] - 0.01:
            new_status = "Overdue"  # Fully unpaid and past due
        else:
            new_status = "Overdue"  # Partially paid, treat as overdue

        frappe.db.sql("""
            UPDATE "tabSales Invoice"
            SET outstanding_amount = %s, status = %s
            WHERE name = %s AND docstatus = 1
        """, (new_outstanding, new_status, m["name"]))
        fixed += 1

    frappe.db.commit()
    print("  Fixed {:,}/{:,}...".format(min(i+batch_size, len(mismatches)), len(mismatches)))

print("Fixed {:,} invoices".format(fixed))

# ═══════════════════════════════════════════════════════════════
# PHASE 5: VERIFY AFTER FIX
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 5: VERIFY AFTER FIX")
print("="*60)

# Problem customer
problem_cust = "CUST-993e9763ce"
after = frappe.db.sql("""
    SELECT COUNT(*) as cnt, COALESCE(SUM(outstanding_amount), 0) as total
    FROM "tabSales Invoice"
    WHERE customer = %s AND docstatus = 1 AND outstanding_amount > 0
""", (problem_cust,), as_dict=True)
print("Problem customer ({}) AFTER fix:".format(problem_cust))
print("  Invoices with outstanding > 0: {}".format(after[0]["cnt"]))
print("  Total outstanding: ${:.2f}".format(float(after[0]["total"])))

# Invoice 607832 specifically
after_607832 = frappe.db.sql('SELECT outstanding_amount, status FROM "tabSales Invoice" WHERE name=%s',
                              ("SINV-607832",), as_dict=True)
if after_607832:
    print("  SINV-607832: outstanding={}, status={}".format(
        after_607832[0]["outstanding_amount"], after_607832[0]["status"]))

# Global stats
global_before_paid = frappe.db.sql("""
    SELECT status, COUNT(*) as cnt, COALESCE(SUM(outstanding_amount), 0) as total
    FROM "tabSales Invoice"
    WHERE docstatus = 1
    GROUP BY status ORDER BY cnt DESC
""", as_dict=True)
print("\nGlobal invoice status distribution AFTER fix:")
for r in global_before_paid:
    print("  {}: {:,} invoices, outstanding ${:,.2f}".format(
        r["status"], r["cnt"], float(r["total"])))

# Double-check: re-scan for remaining mismatches
print("\nRe-scanning for remaining mismatches...")
remaining_mismatches = 0
erp_after = frappe.db.sql("""
    SELECT name, outstanding_amount FROM "tabSales Invoice" WHERE docstatus = 1
""", as_dict=True)
for inv in erp_after:
    try:
        legacy_id = int(inv["name"].split("-")[1])
    except (IndexError, ValueError):
        continue
    leg = legacy.get(legacy_id)
    if not leg:
        continue
    if abs(float(inv["outstanding_amount"] or 0) - leg["montant_du"]) > 0.005:
        remaining_mismatches += 1

print("Remaining mismatches after fix: {}".format(remaining_mismatches))

frappe.clear_cache()
elapsed = time.time() - T_TOTAL
print("\n" + "="*60)
print("DONE in {:.1f}s — cache cleared".format(elapsed))
print("="*60)