gigafibre-fsm/scripts/migration/import_expro_payments.py

"""
Import missing payments for Expro Transit Inc (account 3673).
Creates Payment Entry documents in ERPNext from legacy data.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_expro_payments.py
"""
import frappe
import pymysql
import os
from datetime import datetime

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

conn = pymysql.connect(
    host="10.100.80.100",
    user="facturation",
    password="*******",
    database="gestionclient",
    cursorclass=pymysql.cursors.DictCursor
)

ACCOUNT_ID = 3673
CUSTOMER = "CUST-cbf03814b9"
COMPANY = "TARGO"

# ═══════════════════════════════════════════════════════════════
# STEP 1: Load legacy data
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("STEP 1: LOAD LEGACY DATA")
print("=" * 60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT p.id, p.date_orig, p.amount, p.type, p.reference
        FROM payment p
        WHERE p.account_id = %s
        ORDER BY p.date_orig ASC
    """, (ACCOUNT_ID,))
    legacy_payments = cur.fetchall()

    cur.execute("""
        SELECT pi.payment_id, pi.invoice_id, pi.amount
        FROM payment_item pi
        JOIN payment p ON p.id = pi.payment_id
        WHERE p.account_id = %s
    """, (ACCOUNT_ID,))
    legacy_allocs = cur.fetchall()

conn.close()

# Build allocation map
alloc_map = {}
for a in legacy_allocs:
    pid = a["payment_id"]
    if pid not in alloc_map:
        alloc_map[pid] = []
    alloc_map[pid].append({
        "invoice_id": a["invoice_id"],
        "amount": float(a["amount"] or 0)
    })

print("Legacy payments: {}".format(len(legacy_payments)))
print("Legacy allocations: {}".format(len(legacy_allocs)))

# ═══════════════════════════════════════════════════════════════
# STEP 2: Find which ones already exist
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("STEP 2: FIND MISSING PAYMENTS")
print("=" * 60)

erp_pes = frappe.db.sql("""
    SELECT name FROM "tabPayment Entry" WHERE party = %s
""", (CUSTOMER,), as_dict=True)
erp_pe_ids = set()
for pe in erp_pes:
    try:
        erp_pe_ids.add(int(pe["name"].split("-")[1]))
    except:
        pass

# Check which invoices exist
erp_invs = frappe.db.sql("""
    SELECT name FROM "tabSales Invoice" WHERE customer = %s AND docstatus = 1
""", (CUSTOMER,), as_dict=True)
erp_inv_names = set(i["name"] for i in erp_invs)

to_create = []
skipped_exists = 0
skipped_no_inv = 0

for p in legacy_payments:
    if p["id"] in erp_pe_ids:
        skipped_exists += 1
        continue

    dt = datetime.fromtimestamp(p["date_orig"]).strftime("%Y-%m-%d") if p["date_orig"] else None
    amount = float(p["amount"] or 0)
    ptype = (p["type"] or "").strip()
    ref = (p["reference"] or "").strip()

    # Get allocations and verify invoices exist
    allocations = alloc_map.get(p["id"], [])
    valid_allocs = []
    for a in allocations:
        sinv_name = "SINV-{}".format(a["invoice_id"])
        if sinv_name in erp_inv_names:
            valid_allocs.append({
                "reference_doctype": "Sales Invoice",
                "reference_name": sinv_name,
                "allocated_amount": a["amount"],
            })
        else:
            skipped_no_inv += 1

    to_create.append({
        "name": "PE-{}".format(p["id"]),
        "date": dt,
        "amount": amount,
        "type": ptype,
        "reference": ref,
        "allocations": valid_allocs,
    })

print("Already exists:  {}".format(skipped_exists))
print("TO CREATE:       {}".format(len(to_create)))
print("Allocs skipped (inv not found): {}".format(skipped_no_inv))

# ═══════════════════════════════════════════════════════════════
# STEP 3: Get/create accounts
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("STEP 3: RESOLVE ACCOUNTS")
print("=" * 60)

# Use the same accounts as existing Payment Entries for this customer
existing_pe = frappe.db.sql("""
    SELECT paid_from, paid_to FROM "tabPayment Entry"
    WHERE party = %s AND docstatus = 1 LIMIT 1
""", (CUSTOMER,), as_dict=True)

if existing_pe:
    receivable = existing_pe[0]["paid_from"]
    paid_to = existing_pe[0]["paid_to"]
else:
    receivable = "Comptes clients - T"
    paid_to = "Banque - T"

print("Receivable account (paid_from): {}".format(receivable))
print("Bank account (paid_to): {}".format(paid_to))

if not receivable or not paid_to:
    print("ERROR: Missing accounts!")
    exit()

# ═══════════════════════════════════════════════════════════════
# STEP 4: CREATE PAYMENT ENTRIES VIA DIRECT SQL
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("STEP 4: CREATE PAYMENT ENTRIES")
print("=" * 60)

created = 0
errors = 0

for p in to_create:
    try:
        pe_name = p["name"]
        posting_date = p["date"] or "2012-01-01"
        amount = p["amount"]

        # Insert Payment Entry
        frappe.db.sql("""
            INSERT INTO "tabPayment Entry" (
                name, creation, modified, modified_by, owner, docstatus,
                naming_series, payment_type, posting_date,
                company, party_type, party, party_name,
                paid_from, paid_to, paid_amount, received_amount,
                target_exchange_rate, source_exchange_rate,
                paid_from_account_currency, paid_to_account_currency,
                reference_no, reference_date,
                mode_of_payment, status
            ) VALUES (
                %s, %s, %s, 'Administrator', 'Administrator', 1,
                'ACC-PAY-.YYYY.-', 'Receive', %s,
                %s, 'Customer', %s, %s,
                %s, %s, %s, %s,
                1.0, 1.0,
                'CAD', 'CAD',
                %s, %s,
                %s, 'Submitted'
            )
        """, (
            pe_name, posting_date, posting_date, posting_date,
            COMPANY, CUSTOMER, "Expro Transit Inc.",
            receivable, paid_to, amount, amount,
            p["reference"] or pe_name, posting_date,
            None,
        ))

        # Insert Payment Entry References (allocations)
        for idx, alloc in enumerate(p["allocations"], 1):
            ref_name = "{}-ref-{}".format(pe_name, idx)
            frappe.db.sql("""
                INSERT INTO "tabPayment Entry Reference" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    parent, parentfield, parenttype,
                    reference_doctype, reference_name, allocated_amount,
                    total_amount, outstanding_amount, exchange_rate
                ) VALUES (
                    %s, %s, %s, 'Administrator', 'Administrator', 1, %s,
                    %s, 'references', 'Payment Entry',
                    %s, %s, %s,
                    0, 0, 1.0
                )
            """, (
                ref_name, posting_date, posting_date, idx,
                pe_name,
                alloc["reference_doctype"], alloc["reference_name"], alloc["allocated_amount"],
            ))

        created += 1
    except Exception as e:
        errors += 1
        if errors <= 5:
            print("  ERROR on {}: {}".format(p["name"], str(e)[:100]))

    if created % 50 == 0 and created > 0:
        frappe.db.commit()
        print("  Created {}/{}...".format(created, len(to_create)))

frappe.db.commit()
print("\nCreated: {}".format(created))
print("Errors:  {}".format(errors))

# ═══════════════════════════════════════════════════════════════
# STEP 5: VERIFY
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("STEP 5: VERIFY")
print("=" * 60)

# Count payments now
pe_after = frappe.db.sql("""
    SELECT COUNT(*) as cnt, COALESCE(SUM(paid_amount), 0) as total
    FROM "tabPayment Entry"
    WHERE party = %s AND docstatus = 1
""", (CUSTOMER,), as_dict=True)[0]
print("Payment Entries after import: {} for ${:,.2f}".format(pe_after["cnt"], float(pe_after["total"])))

# Check a few samples
samples = frappe.db.sql("""
    SELECT pe.name, pe.posting_date, pe.paid_amount,
           (SELECT COUNT(*) FROM "tabPayment Entry Reference" per WHERE per.parent = pe.name) as ref_count
    FROM "tabPayment Entry" pe
    WHERE pe.party = %s AND pe.docstatus = 1
    ORDER BY pe.posting_date DESC LIMIT 10
""", (CUSTOMER,), as_dict=True)
print("\nRecent payments:")
for s in samples:
    print("  {} date={} amount={:,.2f} refs={}".format(s["name"], s["posting_date"], float(s["paid_amount"]), s["ref_count"]))

frappe.clear_cache()
print("\nDone — cache cleared")