101faa21f1
- InlineField component + useInlineEdit composable for Odoo-style dblclick editing - Client search by name, account ID, and legacy_customer_id (or_filters) - SMS/Email notification panel on ContactCard via n8n webhooks - Ticket reply thread via Communication docs - All migration scripts (51 files) now tracked - Client portal and field tech app added to monorepo - README rewritten with full feature list, migration summary, architecture - CHANGELOG updated with all recent work - ROADMAP updated with current completion status - Removed hardcoded tokens from docs (use $ERP_SERVICE_TOKEN) - .gitignore updated (docker/, .claude/, exports/, .quasar/) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1285 lines
52 KiB
Python
1285 lines
52 KiB
Python
"""
|
|
Clean reimport: Delete all migration data and reimport from legacy.
|
|
Run inside erpnext-backend-1:
|
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/clean_reimport.py
|
|
|
|
Migrates invoices, payments, GL entries, PLE entries, and credit allocations
|
|
from legacy MariaDB (gestionclient) to ERPNext PostgreSQL.
|
|
|
|
Naming: SINV-{legacy_id} for invoices, PE-{legacy_id} for payments.
|
|
Post-migration naming will use SINV-YYYY-NNNNN format to avoid collisions.
|
|
"""
|
|
import frappe
|
|
import pymysql
|
|
import os
|
|
import time
|
|
import re
|
|
import sys
|
|
|
|
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1) # unbuffered
|
|
|
|
os.chdir("/home/frappe/frappe-bench/sites")
|
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
|
frappe.connect()
|
|
print("Connected:", frappe.local.site)
|
|
|
|
T_TOTAL = time.time()
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 0: CLEANUP — Delete all existing accounting data
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 0: CLEANUP")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
tables_to_clear = [
|
|
("tabGL Entry", ""),
|
|
("tabPayment Ledger Entry", ""),
|
|
("tabSales Taxes and Charges", ""),
|
|
("tabPayment Entry Reference", ""),
|
|
("tabPayment Entry", ""),
|
|
("tabSales Invoice Payment", ""),
|
|
("tabSales Invoice Item", ""),
|
|
("tabSales Invoice", ""),
|
|
]
|
|
|
|
for table, extra in tables_to_clear:
|
|
c = frappe.db.sql('SELECT COUNT(*) FROM "{}"'.format(table))[0][0]
|
|
if c > 0:
|
|
frappe.db.sql('DELETE FROM "{}"'.format(table))
|
|
frappe.db.commit()
|
|
print(" Deleted {} rows from {}".format(c, table))
|
|
else:
|
|
print(" {} already empty".format(table))
|
|
|
|
print(" Cleanup done [{:.0f}s]".format(time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 1: LOAD ALL LEGACY DATA
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 1: LOAD LEGACY DATA")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
legacy = pymysql.connect(
|
|
host="10.100.80.100", user="facturation", password="*******",
|
|
database="gestionclient", cursorclass=pymysql.cursors.DictCursor
|
|
)
|
|
|
|
# 1a: Load invoices
|
|
print("\n--- 1a: Invoices ---")
|
|
with legacy.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT i.id, i.account_id, i.total_amt, i.billed_amt,
|
|
FROM_UNIXTIME(i.date_orig) as date_created,
|
|
i.date_orig, i.notes
|
|
FROM invoice i
|
|
WHERE i.date_orig > 0
|
|
ORDER BY i.id
|
|
""")
|
|
invoices = cur.fetchall()
|
|
print(" Loaded {} invoices".format(len(invoices)))
|
|
|
|
# 1b: Load invoice items
|
|
print("\n--- 1b: Invoice items ---")
|
|
with legacy.cursor() as cur:
|
|
invoice_ids = [str(i['id']) for i in invoices]
|
|
# Process in chunks to avoid query too large
|
|
all_items = {}
|
|
chunk = 10000
|
|
for idx in range(0, len(invoice_ids), chunk):
|
|
batch = invoice_ids[idx:idx+chunk]
|
|
cur.execute("""
|
|
SELECT ii.invoice_id, ii.product_name, ii.quantity, ii.unitary_price,
|
|
(ii.quantity * ii.unitary_price) as total
|
|
FROM invoice_item ii
|
|
WHERE ii.invoice_id IN ({})
|
|
""".format(",".join(batch)))
|
|
for it in cur.fetchall():
|
|
iid = it['invoice_id']
|
|
if iid not in all_items:
|
|
all_items[iid] = []
|
|
all_items[iid].append(it)
|
|
total_items = sum(len(v) for v in all_items.values())
|
|
print(" Loaded {} items for {} invoices".format(total_items, len(all_items)))
|
|
|
|
# 1c: Load invoice taxes
|
|
print("\n--- 1c: Invoice taxes ---")
|
|
with legacy.cursor() as cur:
|
|
all_taxes = {}
|
|
for idx in range(0, len(invoice_ids), chunk):
|
|
batch = invoice_ids[idx:idx+chunk]
|
|
cur.execute("""
|
|
SELECT invoice_id, tax_name, amount
|
|
FROM invoice_tax
|
|
WHERE invoice_id IN ({})
|
|
""".format(",".join(batch)))
|
|
for t in cur.fetchall():
|
|
iid = t['invoice_id']
|
|
if iid not in all_taxes:
|
|
all_taxes[iid] = {'tps': 0, 'tvq': 0}
|
|
name = (t['tax_name'] or '').upper()
|
|
if 'TPS' in name:
|
|
all_taxes[iid]['tps'] = float(t['amount'] or 0)
|
|
elif 'TVQ' in name:
|
|
all_taxes[iid]['tvq'] = float(t['amount'] or 0)
|
|
print(" Loaded {} tax records".format(len(all_taxes)))
|
|
|
|
# 1d: Load payments
|
|
print("\n--- 1d: Payments ---")
|
|
with legacy.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT p.id, p.account_id, p.amount, p.type, p.memo, p.reference,
|
|
FROM_UNIXTIME(p.date_orig) as date_created,
|
|
p.date_orig
|
|
FROM payment p
|
|
WHERE p.date_orig > 0
|
|
AND p.type NOT IN ('credit', 'reversement', 'credit targo')
|
|
ORDER BY p.id
|
|
""")
|
|
payments_raw = cur.fetchall()
|
|
|
|
# Deduplicate payments by (account_id, reference) — legacy double-submits
|
|
seen_refs = set()
|
|
payments = []
|
|
dedup_count = 0
|
|
for p in payments_raw:
|
|
ref = p.get('reference') or ''
|
|
if ref:
|
|
key = (p['account_id'], ref)
|
|
if key in seen_refs:
|
|
dedup_count += 1
|
|
continue
|
|
seen_refs.add(key)
|
|
payments.append(p)
|
|
print(" Loaded {} payments, deduplicated {} (by reference)".format(len(payments), dedup_count))
|
|
|
|
# 1e: Load payment items (allocations)
|
|
print("\n--- 1e: Payment allocations ---")
|
|
with legacy.cursor() as cur:
|
|
payment_ids = [str(p['id']) for p in payments]
|
|
all_payment_items = {}
|
|
for idx in range(0, len(payment_ids), chunk):
|
|
batch = payment_ids[idx:idx+chunk]
|
|
cur.execute("""
|
|
SELECT pi.payment_id, pi.invoice_id, pi.amount
|
|
FROM payment_item pi
|
|
WHERE pi.payment_id IN ({})
|
|
AND pi.amount != 0
|
|
""".format(",".join(batch)))
|
|
for pi in cur.fetchall():
|
|
pid = pi['payment_id']
|
|
if pid not in all_payment_items:
|
|
all_payment_items[pid] = []
|
|
all_payment_items[pid].append(pi)
|
|
total_pi = sum(len(v) for v in all_payment_items.values())
|
|
print(" Loaded {} allocations for {} payments".format(total_pi, len(all_payment_items)))
|
|
|
|
# 1f: Load credit allocations (credit type payments)
|
|
print("\n--- 1f: Credit allocations ---")
|
|
with legacy.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT p.id as payment_id, p.memo, p.account_id,
|
|
pi.invoice_id as target_invoice_id, pi.amount as allocated_amount
|
|
FROM payment p
|
|
JOIN payment_item pi ON pi.payment_id = p.id
|
|
WHERE p.type = 'credit'
|
|
AND p.date_orig > 0
|
|
AND pi.amount != 0
|
|
""")
|
|
credit_allocs_raw = cur.fetchall()
|
|
|
|
credit_allocs = []
|
|
for row in credit_allocs_raw:
|
|
memo = row.get("memo") or ""
|
|
match = re.search(r'#(\d+)', memo)
|
|
if match:
|
|
credit_allocs.append({
|
|
"source_invoice_id": match.group(1),
|
|
"target_invoice_id": str(row["target_invoice_id"]),
|
|
"amount": float(row["allocated_amount"]),
|
|
})
|
|
print(" Loaded {} credit allocations".format(len(credit_allocs)))
|
|
|
|
# 1g: Load reversement allocations (reversement payments link credit → target via memo + payment_item)
|
|
print("\n--- 1g: Reversement allocations ---")
|
|
with legacy.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT p.memo, pi.invoice_id as target_invoice_id
|
|
FROM payment p
|
|
JOIN payment_item pi ON pi.payment_id = p.id
|
|
WHERE p.type = 'reversement'
|
|
AND p.date_orig > 0
|
|
AND p.memo LIKE 'create by invoice #%'
|
|
""")
|
|
rev_allocs_raw = cur.fetchall()
|
|
|
|
reversement_allocs = []
|
|
for row in rev_allocs_raw:
|
|
memo = row.get("memo") or ""
|
|
# memo format: "create by invoice #CREDIT for invoice #TARGET"
|
|
m = re.search(r'create by invoice #(\d+)', memo)
|
|
if m:
|
|
reversement_allocs.append({
|
|
"source_invoice_id": m.group(1),
|
|
"target_invoice_id": str(row["target_invoice_id"]),
|
|
})
|
|
print(" Loaded {} reversement allocations".format(len(reversement_allocs)))
|
|
|
|
legacy.close()
|
|
print("\n All legacy data loaded [{:.0f}s]".format(time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 2: MAP CUSTOMERS
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 2: MAP CUSTOMERS")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
cust_rows = frappe.db.sql("""
|
|
SELECT name, legacy_account_id FROM "tabCustomer"
|
|
WHERE legacy_account_id IS NOT NULL
|
|
""", as_dict=True)
|
|
acct_to_cust = {}
|
|
for c in cust_rows:
|
|
acct_to_cust[str(c["legacy_account_id"])] = c["name"]
|
|
print(" Mapped {} customers".format(len(acct_to_cust)))
|
|
|
|
# Check for unmapped invoices
|
|
unmapped_invoices = []
|
|
for inv in invoices:
|
|
if str(inv['account_id']) not in acct_to_cust:
|
|
unmapped_invoices.append(inv['id'])
|
|
print(" Invoices with unmapped customers: {}".format(len(unmapped_invoices)))
|
|
if unmapped_invoices:
|
|
print(" First 10 unmapped: {}".format(unmapped_invoices[:10]))
|
|
|
|
print(" [{:.0f}s]".format(time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 3: INSERT SALES INVOICES
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 3: INSERT SALES INVOICES")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# Build invoice ID map for credit allocations
|
|
inv_id_to_sinv = {} # legacy_id -> SINV name
|
|
|
|
# Returns are identified by negative total_amt
|
|
# Credit links come from payment type='credit' memo field (loaded in credit_allocs)
|
|
|
|
insert_count = 0
|
|
skip_count = 0
|
|
CHUNK = 2000
|
|
|
|
for idx in range(0, len(invoices), CHUNK):
|
|
batch = invoices[idx:idx+CHUNK]
|
|
si_values = []
|
|
si_item_values = []
|
|
si_tax_values = []
|
|
|
|
for inv in batch:
|
|
cust = acct_to_cust.get(str(inv['account_id']))
|
|
if not cust:
|
|
skip_count += 1
|
|
continue
|
|
|
|
sinv_name = "SINV-{}".format(inv['id'])
|
|
inv_id_to_sinv[str(inv['id'])] = sinv_name
|
|
|
|
total_amt = float(inv['total_amt'])
|
|
is_return = 1 if total_amt < 0 else 0
|
|
posting_date = str(inv['date_created'])[:10]
|
|
|
|
# Tax split
|
|
tax_info = all_taxes.get(inv['id'])
|
|
if tax_info:
|
|
tps = float(tax_info['tps'])
|
|
tvq = float(tax_info['tvq'])
|
|
else:
|
|
# Estimate: total_amt includes 14.975% tax
|
|
if total_amt != 0:
|
|
net = round(total_amt / 1.14975, 2)
|
|
tps = round(net * 0.05, 2)
|
|
tvq = round(net * 0.09975, 2)
|
|
else:
|
|
net = 0
|
|
tps = 0
|
|
tvq = 0
|
|
|
|
net_total = round(total_amt - tps - tvq, 2)
|
|
|
|
# return_against — set later in phase for credit allocations
|
|
return_against = ""
|
|
|
|
si_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"0, '{customer}', '{customer_name}', "
|
|
"'{posting_date}', '{posting_date}', "
|
|
"'{company}', '{currency}', 1, "
|
|
"{grand_total}, {grand_total}, "
|
|
"{net_total}, {net_total}, {net_total}, {net_total}, "
|
|
"{total_taxes}, "
|
|
"{is_return}, '{return_against}', "
|
|
"{outstanding}, "
|
|
"'{debit_to}', 'Customer', "
|
|
"{legacy_id}, "
|
|
"'Draft', 1)".format(
|
|
name=sinv_name,
|
|
customer=cust.replace("'", "''"),
|
|
customer_name=cust.replace("'", "''"),
|
|
posting_date=posting_date,
|
|
company="TARGO",
|
|
currency="CAD",
|
|
grand_total=total_amt,
|
|
net_total=net_total,
|
|
total_taxes=round(tps + tvq, 2),
|
|
is_return=is_return,
|
|
return_against=return_against,
|
|
outstanding=total_amt,
|
|
debit_to="Comptes clients - T",
|
|
legacy_id=inv['id'],
|
|
)
|
|
)
|
|
|
|
# Invoice items
|
|
items = all_items.get(inv['id'], [])
|
|
if not items:
|
|
# Create a single catch-all item
|
|
items = [{"product_name": "Services", "quantity": 1, "unitary_price": net_total, "total": net_total}]
|
|
|
|
for item_idx, it in enumerate(items):
|
|
item_name = "SII-{}-{}".format(inv['id'], item_idx)
|
|
item_total = float(it.get('total', 0) or 0)
|
|
item_qty = float(it.get('quantity', 1) or 1)
|
|
item_rate = float(it.get('unitary_price', 0) or 0)
|
|
desc = str(it.get('product_name', 'Services') or 'Services').replace("'", "''")[:140]
|
|
|
|
si_item_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"'{parent}', 'Sales Invoice', 'items', {idx}, "
|
|
"'{item_code}', '{item_name}', '{description}', '{uom}', "
|
|
"{qty}, {rate}, {amount}, {amount}, {amount}, {amount}, "
|
|
"'{income_account}', 0)".format(
|
|
name=item_name,
|
|
parent=sinv_name,
|
|
idx=item_idx + 1,
|
|
item_code="SVC",
|
|
item_name=desc[:140],
|
|
description=desc[:140],
|
|
uom="Nos",
|
|
qty=item_qty,
|
|
rate=item_rate,
|
|
amount=item_total,
|
|
income_account="Autres produits d''exploitation - T",
|
|
)
|
|
)
|
|
|
|
# Tax rows
|
|
if tps != 0 or tvq != 0:
|
|
si_tax_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"'{parent}', 'Sales Invoice', 'taxes', 1, "
|
|
"'On Net Total', 'Comptes clients - T', "
|
|
"'TPS à payer - T', 5.0, "
|
|
"{tps}, {tps}, {running1}, "
|
|
"'834975559RT0001', 0)".format(
|
|
name="stc-tps-{}".format(inv['id']),
|
|
parent=sinv_name,
|
|
tps=tps,
|
|
running1=round(net_total + tps, 2),
|
|
)
|
|
)
|
|
si_tax_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"'{parent}', 'Sales Invoice', 'taxes', 2, "
|
|
"'On Net Total', 'Comptes clients - T', "
|
|
"'TVQ à payer - T', 9.975, "
|
|
"{tvq}, {tvq}, {running2}, "
|
|
"'1213765929TQ0001', 0)".format(
|
|
name="stc-tvq-{}".format(inv['id']),
|
|
parent=sinv_name,
|
|
tvq=tvq,
|
|
running2=total_amt,
|
|
)
|
|
)
|
|
|
|
insert_count += 1
|
|
|
|
# Bulk INSERT Sales Invoices
|
|
if si_values:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSales Invoice" (
|
|
name, owner, creation, modified, modified_by,
|
|
docstatus, customer, customer_name,
|
|
posting_date, due_date,
|
|
company, currency, conversion_rate,
|
|
grand_total, base_grand_total,
|
|
net_total, base_net_total, total, base_total,
|
|
total_taxes_and_charges,
|
|
is_return, return_against,
|
|
outstanding_amount,
|
|
debit_to, party_account_currency,
|
|
legacy_invoice_id,
|
|
status, set_posting_time
|
|
) VALUES {}
|
|
""".format(",".join(si_values)))
|
|
|
|
# Bulk INSERT Invoice Items
|
|
if si_item_values:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSales Invoice Item" (
|
|
name, owner, creation, modified, modified_by,
|
|
parent, parenttype, parentfield, idx,
|
|
item_code, item_name, description, uom,
|
|
qty, rate, amount, base_amount, net_amount, base_net_amount,
|
|
income_account, docstatus
|
|
) VALUES {}
|
|
""".format(",".join(si_item_values)))
|
|
|
|
# Bulk INSERT Tax rows
|
|
if si_tax_values:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSales Taxes and Charges" (
|
|
name, owner, creation, modified, modified_by,
|
|
parent, parenttype, parentfield, idx,
|
|
charge_type, account_head,
|
|
description, rate,
|
|
tax_amount, base_tax_amount, total,
|
|
cost_center, docstatus
|
|
) VALUES {}
|
|
""".format(",".join(si_tax_values)))
|
|
|
|
frappe.db.commit()
|
|
if (idx + CHUNK) % 20000 == 0 or idx + CHUNK >= len(invoices):
|
|
print(" Inserted {}/{} invoices...".format(min(idx + CHUNK, len(invoices)), len(invoices)))
|
|
|
|
print(" Inserted {} invoices, skipped {} (no customer) [{:.0f}s]".format(insert_count, skip_count, time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 4: SET return_against AND UPDATE docstatus
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 4: SET return_against + SUBMIT ALL")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# Set return_against from credit_allocs (credit payments) — batch via temp table
|
|
credit_alloc_pairs = []
|
|
credit_alloc_source_ids = set()
|
|
for alloc in credit_allocs:
|
|
source_sinv = inv_id_to_sinv.get(alloc["source_invoice_id"])
|
|
target_sinv = inv_id_to_sinv.get(alloc["target_invoice_id"])
|
|
if source_sinv and target_sinv and alloc["source_invoice_id"] not in credit_alloc_source_ids:
|
|
credit_alloc_pairs.append((source_sinv, target_sinv))
|
|
credit_alloc_source_ids.add(alloc["source_invoice_id"])
|
|
|
|
if credit_alloc_pairs:
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
frappe.db.sql("CREATE TABLE _tmp_return_against (source_sinv VARCHAR(140), target_sinv VARCHAR(140))")
|
|
frappe.db.commit()
|
|
for i in range(0, len(credit_alloc_pairs), 5000):
|
|
batch = credit_alloc_pairs[i:i+5000]
|
|
values = ",".join(["('{}', '{}')".format(s, t) for s, t in batch])
|
|
frappe.db.sql("INSERT INTO _tmp_return_against VALUES {}".format(values))
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice" si
|
|
SET return_against = ra.target_sinv
|
|
FROM _tmp_return_against ra
|
|
WHERE si.name = ra.source_sinv
|
|
AND si.is_return = 1
|
|
AND (si.return_against IS NULL OR si.return_against = '')
|
|
""")
|
|
frappe.db.commit()
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
print(" Set return_against from credit allocs on {} invoices".format(len(credit_alloc_pairs)))
|
|
|
|
# Set return_against from reversal notes (Renversement de la facture #NNN)
|
|
# These are SEPARATE from credit allocs — zero overlap
|
|
reversal_matches = [] # (credit_sinv, target_sinv)
|
|
for inv in invoices:
|
|
if float(inv['total_amt']) >= 0:
|
|
continue
|
|
notes = inv.get('notes') or ''
|
|
if 'Renversement de la facture #' not in notes:
|
|
continue
|
|
m = re.search(r'#(\d+)', notes)
|
|
if not m:
|
|
continue
|
|
target_leg_id = m.group(1)
|
|
source_sinv = inv_id_to_sinv.get(str(inv['id']))
|
|
target_sinv = inv_id_to_sinv.get(target_leg_id)
|
|
if source_sinv and target_sinv and str(inv['id']) not in credit_alloc_source_ids:
|
|
reversal_matches.append((source_sinv, target_sinv))
|
|
|
|
if reversal_matches:
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
frappe.db.sql("CREATE TABLE _tmp_return_against (source_sinv VARCHAR(140), target_sinv VARCHAR(140))")
|
|
frappe.db.commit()
|
|
for i in range(0, len(reversal_matches), 5000):
|
|
batch = reversal_matches[i:i+5000]
|
|
values = ",".join(["('{}', '{}')".format(s, t) for s, t in batch])
|
|
frappe.db.sql("INSERT INTO _tmp_return_against VALUES {}".format(values))
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice" si
|
|
SET return_against = ra.target_sinv
|
|
FROM _tmp_return_against ra
|
|
WHERE si.name = ra.source_sinv
|
|
AND si.is_return = 1
|
|
AND (si.return_against IS NULL OR si.return_against = '')
|
|
""")
|
|
frappe.db.commit()
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
print(" Set return_against from reversal notes on {} invoices".format(len(reversal_matches)))
|
|
|
|
# Set return_against from reversement payment memos (3rd mechanism)
|
|
# memo: "create by invoice #CREDIT for invoice #TARGET", payment_item.invoice_id = TARGET
|
|
rev_notes_source_ids = set(str(inv['id']) for inv in invoices
|
|
if float(inv['total_amt']) < 0 and (inv.get('notes') or '').startswith('Renversement'))
|
|
already_linked = credit_alloc_source_ids | rev_notes_source_ids
|
|
|
|
reversement_matches = []
|
|
for alloc in reversement_allocs:
|
|
if alloc["source_invoice_id"] in already_linked:
|
|
continue
|
|
source_sinv = inv_id_to_sinv.get(alloc["source_invoice_id"])
|
|
target_sinv = inv_id_to_sinv.get(alloc["target_invoice_id"])
|
|
if source_sinv and target_sinv:
|
|
reversement_matches.append((source_sinv, target_sinv))
|
|
already_linked.add(alloc["source_invoice_id"])
|
|
|
|
if reversement_matches:
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
frappe.db.sql("CREATE TABLE _tmp_return_against (source_sinv VARCHAR(140), target_sinv VARCHAR(140))")
|
|
frappe.db.commit()
|
|
for i in range(0, len(reversement_matches), 5000):
|
|
batch = reversement_matches[i:i+5000]
|
|
values = ",".join(["('{}', '{}')".format(s, t) for s, t in batch])
|
|
frappe.db.sql("INSERT INTO _tmp_return_against VALUES {}".format(values))
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice" si
|
|
SET return_against = ra.target_sinv
|
|
FROM _tmp_return_against ra
|
|
WHERE si.name = ra.source_sinv
|
|
AND si.is_return = 1
|
|
AND (si.return_against IS NULL OR si.return_against = '')
|
|
""")
|
|
frappe.db.commit()
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_return_against")
|
|
frappe.db.commit()
|
|
print(" Set return_against from reversement memos on {} invoices".format(len(reversement_matches)))
|
|
|
|
# Submit all invoices
|
|
frappe.db.sql('UPDATE "tabSales Invoice" SET docstatus = 1')
|
|
frappe.db.sql('UPDATE "tabSales Invoice Item" SET docstatus = 1')
|
|
frappe.db.sql('UPDATE "tabSales Taxes and Charges" SET docstatus = 1')
|
|
frappe.db.commit()
|
|
sinv_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSales Invoice" WHERE docstatus = 1')[0][0]
|
|
print(" Submitted {} invoices [{:.0f}s]".format(sinv_count, time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 5: INSERT PAYMENT ENTRIES
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 5: INSERT PAYMENT ENTRIES")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
pe_count = 0
|
|
per_count = 0
|
|
|
|
for idx in range(0, len(payments), CHUNK):
|
|
batch = payments[idx:idx+CHUNK]
|
|
pe_values = []
|
|
per_values = []
|
|
|
|
for pmt in batch:
|
|
cust = acct_to_cust.get(str(pmt['account_id']))
|
|
if not cust:
|
|
continue
|
|
|
|
pe_name = "PE-{}".format(pmt['id'])
|
|
amount = float(pmt['amount'])
|
|
posting_date = str(pmt['date_created'])[:10]
|
|
pmt_type = pmt.get('type', 'payment')
|
|
memo = str(pmt.get('memo', '') or '')[:140].replace("'", "''").replace("\\", "\\\\")
|
|
|
|
pe_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"1, '{posting_date}', 'TARGO', "
|
|
"'Receive', 'Customer', '{party}', '{party_name}', "
|
|
"'Comptes clients - T', 'Banque - T', "
|
|
"{paid}, {paid}, {paid}, {received}, "
|
|
"1, 'CAD', 'CAD', "
|
|
"'{remarks}', "
|
|
"'Submitted', {legacy_id})".format(
|
|
name=pe_name,
|
|
posting_date=posting_date,
|
|
party=cust.replace("'", "''"),
|
|
party_name=cust.replace("'", "''"),
|
|
paid=amount,
|
|
received=amount,
|
|
remarks=memo,
|
|
legacy_id=pmt['id'],
|
|
)
|
|
)
|
|
pe_count += 1
|
|
|
|
# Payment references (allocations to invoices)
|
|
items = all_payment_items.get(pmt['id'], [])
|
|
for pi_idx, pi in enumerate(items):
|
|
target_sinv = inv_id_to_sinv.get(str(pi['invoice_id']))
|
|
if not target_sinv:
|
|
continue
|
|
per_name = "PER-{}-{}".format(pmt['id'], pi_idx)
|
|
alloc_amt = float(pi['amount'])
|
|
|
|
per_values.append(
|
|
"('{name}', 'Administrator', NOW(), NOW(), 'Administrator', "
|
|
"'{parent}', 'Payment Entry', 'references', {idx}, "
|
|
"'Sales Invoice', '{ref_name}', "
|
|
"{allocated}, "
|
|
"1, 1)".format(
|
|
name=per_name,
|
|
parent=pe_name,
|
|
idx=pi_idx + 1,
|
|
ref_name=target_sinv,
|
|
allocated=alloc_amt,
|
|
)
|
|
)
|
|
per_count += 1
|
|
|
|
if pe_values:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Entry" (
|
|
name, owner, creation, modified, modified_by,
|
|
docstatus, posting_date, company,
|
|
payment_type, party_type, party, party_name,
|
|
paid_from, paid_to,
|
|
paid_amount, base_paid_amount, received_amount, base_received_amount,
|
|
target_exchange_rate, paid_from_account_currency, paid_to_account_currency,
|
|
remarks,
|
|
status, legacy_payment_id
|
|
) VALUES {}
|
|
""".format(",".join(pe_values)))
|
|
|
|
if per_values:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Entry Reference" (
|
|
name, owner, creation, modified, modified_by,
|
|
parent, parenttype, parentfield, idx,
|
|
reference_doctype, reference_name,
|
|
allocated_amount,
|
|
exchange_rate, docstatus
|
|
) VALUES {}
|
|
""".format(",".join(per_values)))
|
|
|
|
frappe.db.commit()
|
|
if (idx + CHUNK) % 20000 == 0 or idx + CHUNK >= len(payments):
|
|
print(" Inserted {}/{} payments...".format(min(idx + CHUNK, len(payments)), len(payments)))
|
|
|
|
print(" {} Payment Entries, {} references [{:.0f}s]".format(pe_count, per_count, time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 6: GL ENTRIES — Invoices (receivable + income + TPS + TVQ)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 6: GL ENTRIES — INVOICES")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# Receivable side (debit for positive invoices, credit for returns)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, party_type, party, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'gir-' || si.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM si.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM si.posting_date)::text || '-' || (EXTRACT(YEAR FROM si.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM si.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM si.posting_date)::text
|
|
END,
|
|
'Comptes clients - T', 'Customer', si.customer, 'CAD',
|
|
GREATEST(si.grand_total, 0), GREATEST(si.grand_total, 0),
|
|
GREATEST(-si.grand_total, 0), GREATEST(-si.grand_total, 0),
|
|
'Sales Invoice', si.name, 'Autres produits d''exploitation - T',
|
|
'No', 0
|
|
FROM "tabSales Invoice" si
|
|
WHERE si.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
gl_recv = frappe.db.sql("SELECT COUNT(*) FROM \"tabGL Entry\" WHERE name LIKE 'gir-%%'")[0][0]
|
|
print(" Receivable GL entries: {}".format(gl_recv))
|
|
|
|
# Income side
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'gii-' || si.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM si.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM si.posting_date)::text || '-' || (EXTRACT(YEAR FROM si.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM si.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM si.posting_date)::text
|
|
END,
|
|
'Autres produits d''exploitation - T', 'CAD',
|
|
GREATEST(-si.net_total, 0), GREATEST(-si.net_total, 0),
|
|
GREATEST(si.net_total, 0), GREATEST(si.net_total, 0),
|
|
'Sales Invoice', si.name, 'Comptes clients - T',
|
|
'No', 0
|
|
FROM "tabSales Invoice" si
|
|
WHERE si.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
gl_inc = frappe.db.sql("SELECT COUNT(*) FROM \"tabGL Entry\" WHERE name LIKE 'gii-%%'")[0][0]
|
|
print(" Income GL entries: {}".format(gl_inc))
|
|
|
|
# TPS GL entries
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'glt-' || stc.parent, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM si.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM si.posting_date)::text || '-' || (EXTRACT(YEAR FROM si.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM si.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM si.posting_date)::text
|
|
END,
|
|
'TPS à payer - T', 'CAD',
|
|
GREATEST(-stc.tax_amount, 0), GREATEST(-stc.tax_amount, 0),
|
|
GREATEST(stc.tax_amount, 0), GREATEST(stc.tax_amount, 0),
|
|
'Sales Invoice', si.name, 'Comptes clients - T',
|
|
'No', 0
|
|
FROM "tabSales Taxes and Charges" stc
|
|
JOIN "tabSales Invoice" si ON si.name = stc.parent
|
|
WHERE si.docstatus = 1 AND stc.description = 'TPS à payer - T'
|
|
""")
|
|
frappe.db.commit()
|
|
gl_tps = frappe.db.sql("SELECT COUNT(*) FROM \"tabGL Entry\" WHERE name LIKE 'glt-%%'")[0][0]
|
|
print(" TPS GL entries: {}".format(gl_tps))
|
|
|
|
# TVQ GL entries
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'glq-' || stc.parent, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM si.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM si.posting_date)::text || '-' || (EXTRACT(YEAR FROM si.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM si.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM si.posting_date)::text
|
|
END,
|
|
'TVQ à payer - T', 'CAD',
|
|
GREATEST(-stc.tax_amount, 0), GREATEST(-stc.tax_amount, 0),
|
|
GREATEST(stc.tax_amount, 0), GREATEST(stc.tax_amount, 0),
|
|
'Sales Invoice', si.name, 'Comptes clients - T',
|
|
'No', 0
|
|
FROM "tabSales Taxes and Charges" stc
|
|
JOIN "tabSales Invoice" si ON si.name = stc.parent
|
|
WHERE si.docstatus = 1 AND stc.description = 'TVQ à payer - T'
|
|
""")
|
|
frappe.db.commit()
|
|
gl_tvq = frappe.db.sql("SELECT COUNT(*) FROM \"tabGL Entry\" WHERE name LIKE 'glq-%%'")[0][0]
|
|
print(" TVQ GL entries: {}".format(gl_tvq))
|
|
|
|
print(" Invoice GL total: {} [{:.0f}s]".format(gl_recv + gl_inc + gl_tps + gl_tvq, time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 7: GL ENTRIES — Payments (bank + receivable)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 7: GL ENTRIES — PAYMENTS")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# Bank side (debit bank)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'gpb-' || pe.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
pe.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM pe.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM pe.posting_date)::text || '-' || (EXTRACT(YEAR FROM pe.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM pe.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM pe.posting_date)::text
|
|
END,
|
|
'Banque - T', 'CAD',
|
|
pe.paid_amount, pe.paid_amount, 0, 0,
|
|
'Payment Entry', pe.name, 'Comptes clients - T',
|
|
'No', 0
|
|
FROM "tabPayment Entry" pe
|
|
WHERE pe.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
# Receivable side (credit receivable)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabGL Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company, fiscal_year,
|
|
account, party_type, party, account_currency,
|
|
debit_in_account_currency, debit, credit_in_account_currency, credit,
|
|
voucher_type, voucher_no, against,
|
|
is_opening, is_cancelled
|
|
)
|
|
SELECT
|
|
'gpr-' || pe.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
pe.posting_date, 'TARGO',
|
|
CASE
|
|
WHEN EXTRACT(MONTH FROM pe.posting_date) >= 7
|
|
THEN EXTRACT(YEAR FROM pe.posting_date)::text || '-' || (EXTRACT(YEAR FROM pe.posting_date) + 1)::text
|
|
ELSE (EXTRACT(YEAR FROM pe.posting_date) - 1)::text || '-' || EXTRACT(YEAR FROM pe.posting_date)::text
|
|
END,
|
|
'Comptes clients - T', 'Customer', pe.party, 'CAD',
|
|
0, 0, pe.paid_amount, pe.paid_amount,
|
|
'Payment Entry', pe.name, 'Banque - T',
|
|
'No', 0
|
|
FROM "tabPayment Entry" pe
|
|
WHERE pe.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
gl_pmt = frappe.db.sql("SELECT COUNT(*) FROM \"tabGL Entry\" WHERE name LIKE 'gpb-%%' OR name LIKE 'gpr-%%'")[0][0]
|
|
print(" Payment GL entries: {} [{:.0f}s]".format(gl_pmt, time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 8: PAYMENT LEDGER ENTRIES
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 8: PAYMENT LEDGER ENTRIES")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# PLE for invoices (self-referencing: against_voucher = self)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Ledger Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company,
|
|
account_type, account, account_currency,
|
|
party_type, party,
|
|
due_date,
|
|
voucher_type, voucher_no,
|
|
against_voucher_type, against_voucher_no,
|
|
amount, amount_in_account_currency,
|
|
delinked, remarks
|
|
)
|
|
SELECT
|
|
'ple-' || si.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
'Receivable', 'Comptes clients - T', 'CAD',
|
|
'Customer', si.customer,
|
|
COALESCE(si.due_date, si.posting_date),
|
|
'Sales Invoice', si.name,
|
|
'Sales Invoice', si.name,
|
|
si.grand_total, si.grand_total,
|
|
0, 'Invoice posting'
|
|
FROM "tabSales Invoice" si
|
|
WHERE si.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
ple_inv = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Ledger Entry\" WHERE name LIKE 'ple-SINV-%%'")[0][0]
|
|
print(" Invoice PLE: {}".format(ple_inv))
|
|
|
|
# PLE for allocated payment references
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Ledger Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company,
|
|
account_type, account, account_currency,
|
|
party_type, party,
|
|
due_date,
|
|
voucher_type, voucher_no,
|
|
against_voucher_type, against_voucher_no,
|
|
amount, amount_in_account_currency,
|
|
delinked, remarks
|
|
)
|
|
SELECT
|
|
'ple-' || per.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
pe.posting_date, 'TARGO',
|
|
'Receivable', 'Comptes clients - T', 'CAD',
|
|
'Customer', pe.party,
|
|
pe.posting_date,
|
|
'Payment Entry', pe.name,
|
|
'Sales Invoice', per.reference_name,
|
|
-per.allocated_amount, -per.allocated_amount,
|
|
0, 'Payment allocation'
|
|
FROM "tabPayment Entry Reference" per
|
|
JOIN "tabPayment Entry" pe ON pe.name = per.parent
|
|
WHERE pe.docstatus = 1 AND per.reference_doctype = 'Sales Invoice'
|
|
""")
|
|
frappe.db.commit()
|
|
ple_per = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Ledger Entry\" WHERE name LIKE 'ple-PER-%%'")[0][0]
|
|
print(" Payment allocation PLE: {}".format(ple_per))
|
|
|
|
# PLE for unallocated payments (no references)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Ledger Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company,
|
|
account_type, account, account_currency,
|
|
party_type, party,
|
|
due_date,
|
|
voucher_type, voucher_no,
|
|
against_voucher_type, against_voucher_no,
|
|
amount, amount_in_account_currency,
|
|
delinked, remarks
|
|
)
|
|
SELECT
|
|
'ple-' || pe.name, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
pe.posting_date, 'TARGO',
|
|
'Receivable', 'Comptes clients - T', 'CAD',
|
|
'Customer', pe.party,
|
|
pe.posting_date,
|
|
'Payment Entry', pe.name,
|
|
'Payment Entry', pe.name,
|
|
-pe.paid_amount, -pe.paid_amount,
|
|
0, 'Unallocated payment'
|
|
FROM "tabPayment Entry" pe
|
|
WHERE pe.docstatus = 1
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM "tabPayment Entry Reference" per
|
|
WHERE per.parent = pe.name
|
|
)
|
|
""")
|
|
frappe.db.commit()
|
|
ple_unalloc = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Ledger Entry\" WHERE remarks = 'Unallocated payment'")[0][0]
|
|
print(" Unallocated payment PLE: {}".format(ple_unalloc))
|
|
|
|
# PLE for credit allocations (credit note → target invoice)
|
|
print("\n Creating credit allocation PLE...")
|
|
credit_ple_count = 0
|
|
for alloc in credit_allocs:
|
|
source_sinv = inv_id_to_sinv.get(alloc["source_invoice_id"])
|
|
target_sinv = inv_id_to_sinv.get(alloc["target_invoice_id"])
|
|
if source_sinv and target_sinv:
|
|
credit_ple_count += 1
|
|
|
|
# Use temp table for batch insert
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_credit_alloc")
|
|
frappe.db.commit()
|
|
frappe.db.sql("""
|
|
CREATE TABLE _tmp_credit_alloc (
|
|
id SERIAL PRIMARY KEY,
|
|
source_sinv VARCHAR(140),
|
|
target_sinv VARCHAR(140),
|
|
amount DOUBLE PRECISION
|
|
)
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
resolved_credits = []
|
|
for alloc in credit_allocs:
|
|
source_sinv = inv_id_to_sinv.get(alloc["source_invoice_id"])
|
|
target_sinv = inv_id_to_sinv.get(alloc["target_invoice_id"])
|
|
if source_sinv and target_sinv:
|
|
resolved_credits.append((source_sinv, target_sinv, alloc["amount"]))
|
|
|
|
for idx in range(0, len(resolved_credits), 5000):
|
|
batch = resolved_credits[idx:idx+5000]
|
|
values = ",".join(["('{}', '{}', {})".format(s, t, a) for s, t, a in batch])
|
|
frappe.db.sql("INSERT INTO _tmp_credit_alloc (source_sinv, target_sinv, amount) VALUES {}".format(values))
|
|
frappe.db.commit()
|
|
|
|
# Delete self-referencing PLE for credit invoices that have allocations
|
|
frappe.db.sql("""
|
|
DELETE FROM "tabPayment Ledger Entry"
|
|
WHERE name IN (
|
|
SELECT 'ple-' || ca.source_sinv
|
|
FROM _tmp_credit_alloc ca
|
|
)
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
# Insert credit allocation PLE
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Ledger Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company,
|
|
account_type, account, account_currency,
|
|
party_type, party,
|
|
due_date,
|
|
voucher_type, voucher_no,
|
|
against_voucher_type, against_voucher_no,
|
|
amount, amount_in_account_currency,
|
|
delinked, remarks
|
|
)
|
|
SELECT
|
|
'plc-' || ca.id, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
'Receivable', 'Comptes clients - T', 'CAD',
|
|
'Customer', si.customer,
|
|
COALESCE(si.due_date, si.posting_date),
|
|
'Sales Invoice', ca.source_sinv,
|
|
'Sales Invoice', ca.target_sinv,
|
|
-ca.amount, -ca.amount,
|
|
0, 'Credit allocation from return invoice'
|
|
FROM _tmp_credit_alloc ca
|
|
JOIN "tabSales Invoice" si ON si.name = ca.source_sinv
|
|
""")
|
|
frappe.db.commit()
|
|
ple_credit = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Ledger Entry\" WHERE name LIKE 'plc-%%'")[0][0]
|
|
print(" Credit allocation PLE: {}".format(ple_credit))
|
|
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_credit_alloc")
|
|
frappe.db.commit()
|
|
|
|
# PLE for reversal invoices (matched via notes OR reversement memos, NOT credit payments)
|
|
all_reversal_matches = reversal_matches + reversement_matches
|
|
if all_reversal_matches:
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_reversal_match")
|
|
frappe.db.commit()
|
|
frappe.db.sql("""
|
|
CREATE TABLE _tmp_reversal_match (
|
|
id SERIAL PRIMARY KEY,
|
|
credit_sinv VARCHAR(140),
|
|
target_sinv VARCHAR(140)
|
|
)
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
for idx_r in range(0, len(all_reversal_matches), 5000):
|
|
batch = all_reversal_matches[idx_r:idx_r+5000]
|
|
values = ",".join(["('{}', '{}')".format(c, t) for c, t in batch])
|
|
frappe.db.sql("INSERT INTO _tmp_reversal_match (credit_sinv, target_sinv) VALUES {}".format(values))
|
|
frappe.db.commit()
|
|
|
|
# Delete self-referencing PLE for reversal credit invoices (only actual returns)
|
|
frappe.db.sql("""
|
|
DELETE FROM "tabPayment Ledger Entry"
|
|
WHERE name IN (
|
|
SELECT 'ple-' || rm.credit_sinv FROM _tmp_reversal_match rm
|
|
JOIN "tabSales Invoice" si ON si.name = rm.credit_sinv
|
|
WHERE si.is_return = 1
|
|
)
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
# Insert reversal PLE (credit invoice → target invoice)
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabPayment Ledger Entry" (
|
|
name, owner, creation, modified, modified_by, docstatus,
|
|
posting_date, company,
|
|
account_type, account, account_currency,
|
|
party_type, party,
|
|
due_date,
|
|
voucher_type, voucher_no,
|
|
against_voucher_type, against_voucher_no,
|
|
amount, amount_in_account_currency,
|
|
delinked, remarks
|
|
)
|
|
SELECT
|
|
'plr-' || rm.id, 'Administrator', NOW(), NOW(), 'Administrator', 1,
|
|
si.posting_date, 'TARGO',
|
|
'Receivable', 'Comptes clients - T', 'CAD',
|
|
'Customer', si.customer,
|
|
COALESCE(si.due_date, si.posting_date),
|
|
'Sales Invoice', rm.credit_sinv,
|
|
'Sales Invoice', rm.target_sinv,
|
|
si.grand_total, si.grand_total,
|
|
0, 'Reversal allocation (from notes)'
|
|
FROM _tmp_reversal_match rm
|
|
JOIN "tabSales Invoice" si ON si.name = rm.credit_sinv
|
|
WHERE si.is_return = 1
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
frappe.db.sql("DROP TABLE IF EXISTS _tmp_reversal_match")
|
|
frappe.db.commit()
|
|
|
|
ple_reversal = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Ledger Entry\" WHERE name LIKE 'plr-%%'")[0][0]
|
|
print(" Reversal PLE: {}".format(ple_reversal))
|
|
|
|
print(" Total PLE: {} [{:.0f}s]".format(
|
|
ple_inv + ple_per + ple_unalloc + ple_credit + ple_reversal,
|
|
time.time() - t0
|
|
))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 9: OUTSTANDING + STATUS
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 9: OUTSTANDING + STATUS")
|
|
print("="*60)
|
|
t0 = time.time()
|
|
|
|
# Outstanding = SUM of all PLE against this invoice
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice" si
|
|
SET outstanding_amount = COALESCE((
|
|
SELECT SUM(ple.amount)
|
|
FROM "tabPayment Ledger Entry" ple
|
|
WHERE ple.against_voucher_type = 'Sales Invoice'
|
|
AND ple.against_voucher_no = si.name
|
|
AND ple.delinked = 0
|
|
), si.grand_total)
|
|
WHERE si.docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
print(" Outstanding amounts calculated")
|
|
|
|
# All return invoices should have outstanding = 0 (settled by definition)
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice"
|
|
SET outstanding_amount = 0
|
|
WHERE is_return = 1 AND docstatus = 1
|
|
""")
|
|
frappe.db.commit()
|
|
|
|
# Fix remaining overpaid non-return invoices (legacy double-settlement: paid + reversed)
|
|
# These were settled in legacy — mark as paid with a correction note
|
|
overpaid_fix = frappe.db.sql("""
|
|
SELECT name, outstanding_amount FROM "tabSales Invoice"
|
|
WHERE docstatus = 1 AND is_return != 1 AND outstanding_amount < -0.005
|
|
""", as_dict=True)
|
|
if overpaid_fix:
|
|
overpaid_names = [r['name'] for r in overpaid_fix]
|
|
overpaid_total = sum(float(r['outstanding_amount']) for r in overpaid_fix)
|
|
# Delink the extra reversal PLE entries that caused overpayment
|
|
frappe.db.sql("""
|
|
UPDATE "tabPayment Ledger Entry"
|
|
SET delinked = 1, remarks = 'Delinked: legacy double-settlement (paid + reversed)'
|
|
WHERE name LIKE 'plr-%%'
|
|
AND against_voucher_no IN ({})
|
|
""".format(",".join(["'{}'".format(n) for n in overpaid_names])))
|
|
frappe.db.commit()
|
|
# Recalculate outstanding on these invoices
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice" si
|
|
SET outstanding_amount = COALESCE((
|
|
SELECT SUM(ple.amount)
|
|
FROM "tabPayment Ledger Entry" ple
|
|
WHERE ple.against_voucher_type = 'Sales Invoice'
|
|
AND ple.against_voucher_no = si.name
|
|
AND ple.delinked = 0
|
|
), si.grand_total)
|
|
WHERE si.name IN ({})
|
|
""".format(",".join(["'{}'".format(n) for n in overpaid_names])))
|
|
frappe.db.commit()
|
|
# Any still negative (e.g. rounding, double credit alloc) — force to 0
|
|
frappe.db.sql("""
|
|
UPDATE "tabSales Invoice"
|
|
SET outstanding_amount = 0
|
|
WHERE docstatus = 1 AND is_return != 1 AND outstanding_amount < -0.005
|
|
""")
|
|
frappe.db.commit()
|
|
print(" Fixed {} overpaid invoices (${:.2f} total) — legacy double-settlement".format(
|
|
len(overpaid_fix), overpaid_total))
|
|
|
|
# Statuses
|
|
frappe.db.sql("""UPDATE "tabSales Invoice" SET status = 'Return' WHERE docstatus = 1 AND is_return = 1""")
|
|
frappe.db.sql("""UPDATE "tabSales Invoice" SET status = 'Paid' WHERE docstatus = 1 AND is_return != 1 AND ROUND(outstanding_amount, 2) = 0""")
|
|
frappe.db.sql("""UPDATE "tabSales Invoice" SET status = 'Overdue' WHERE docstatus = 1 AND is_return != 1 AND outstanding_amount > 0.005 AND COALESCE(due_date, posting_date) < CURRENT_DATE""")
|
|
frappe.db.sql("""UPDATE "tabSales Invoice" SET status = 'Unpaid' WHERE docstatus = 1 AND is_return != 1 AND outstanding_amount > 0.005 AND COALESCE(due_date, posting_date) >= CURRENT_DATE""")
|
|
frappe.db.sql("""UPDATE "tabSales Invoice" SET status = 'Credit Note Issued' WHERE docstatus = 1 AND is_return != 1 AND outstanding_amount < -0.005""")
|
|
frappe.db.commit()
|
|
print(" Statuses updated [{:.0f}s]".format(time.time() - t0))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# VERIFICATION
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("VERIFICATION")
|
|
print("="*60)
|
|
|
|
# Counts
|
|
si_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSales Invoice" WHERE docstatus = 1')[0][0]
|
|
pe_final = frappe.db.sql('SELECT COUNT(*) FROM "tabPayment Entry" WHERE docstatus = 1')[0][0]
|
|
gl_count = frappe.db.sql('SELECT COUNT(*) FROM "tabGL Entry"')[0][0]
|
|
ple_count = frappe.db.sql('SELECT COUNT(*) FROM "tabPayment Ledger Entry"')[0][0]
|
|
print(" Sales Invoices: {}".format(si_count))
|
|
print(" Payment Entries: {}".format(pe_final))
|
|
print(" GL Entries: {}".format(gl_count))
|
|
print(" PLE Entries: {}".format(ple_count))
|
|
|
|
# GL Balance
|
|
gl_balance = frappe.db.sql("""
|
|
SELECT
|
|
ROUND(SUM(debit), 2) as total_debit,
|
|
ROUND(SUM(credit), 2) as total_credit,
|
|
ROUND(SUM(debit) - SUM(credit), 2) as diff
|
|
FROM "tabGL Entry"
|
|
""", as_dict=True)[0]
|
|
print("\n GL Balance: debit=${} credit=${} diff=${}".format(
|
|
gl_balance['total_debit'], gl_balance['total_credit'], gl_balance['diff']))
|
|
|
|
# Status breakdown
|
|
statuses = frappe.db.sql("""
|
|
SELECT status, COUNT(*) as cnt
|
|
FROM "tabSales Invoice" WHERE docstatus = 1
|
|
GROUP BY status ORDER BY cnt DESC
|
|
""", as_dict=True)
|
|
print("\n Invoice status breakdown:")
|
|
for s in statuses:
|
|
print(" {}: {}".format(s['status'], s['cnt']))
|
|
|
|
# Outstanding totals
|
|
outstanding = frappe.db.sql("""
|
|
SELECT
|
|
ROUND(SUM(CASE WHEN outstanding_amount > 0.005 THEN outstanding_amount ELSE 0 END), 2) as owed,
|
|
ROUND(SUM(CASE WHEN outstanding_amount < -0.005 THEN outstanding_amount ELSE 0 END), 2) as overpaid
|
|
FROM "tabSales Invoice" WHERE docstatus = 1
|
|
""", as_dict=True)[0]
|
|
print("\n Outstanding: ${} owed | ${} overpaid".format(outstanding['owed'], outstanding['overpaid']))
|
|
|
|
# Check invoice 638567
|
|
inv_check = frappe.db.sql("""
|
|
SELECT name, grand_total, outstanding_amount, status, customer
|
|
FROM "tabSales Invoice" WHERE legacy_invoice_id = 638567
|
|
""", as_dict=True)
|
|
if inv_check:
|
|
i = inv_check[0]
|
|
print("\n Invoice 638567: {} | total={} | outstanding={} | {} | {}".format(
|
|
i['name'], i['grand_total'], i['outstanding_amount'], i['status'], i['customer']))
|
|
else:
|
|
print("\n Invoice 638567: NOT FOUND (customer may not be mapped)")
|
|
|
|
elapsed = time.time() - T_TOTAL
|
|
print("\n" + "="*60)
|
|
print("CLEAN REIMPORT COMPLETE in {:.0f}s".format(elapsed))
|
|
print("="*60)
|