#!/usr/bin/env python3 """ Reconcile Service Subscription (complete legacy import) vs Subscription (ERPNext native). Identifies per-customer discrepancies: - Missing records in Subscription that exist in Service Subscription - Price/status mismatches - Generates a sync plan to fix the native Subscription table Run inside erpnext-backend-1: /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reconcile_subscriptions.py Modes: --audit Audit only — report discrepancies (default) --sync Create missing Subscriptions from Service Subscriptions --fix Fix price/status mismatches on existing Subscriptions --full Audit + Sync + Fix (do everything) """ import sys import os import json from datetime import datetime, timezone import uuid os.chdir("/home/frappe/frappe-bench/sites") import frappe frappe.init(site="erp.gigafibre.ca", sites_path=".") frappe.connect() frappe.local.flags.ignore_permissions = True print(f"Connected: {frappe.local.site}") ADMIN = "Administrator" COMPANY = "TARGO" TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T" def uid(prefix=""): return prefix + uuid.uuid4().hex[:10] def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") # ═══════════════════════════════════════════════════════════════ # STEP 1: Load all Service Subscriptions (source of truth) # ═══════════════════════════════════════════════════════════════ def load_service_subscriptions(): """Load all Service Subscriptions grouped by customer.""" print("\n" + "=" * 70) print("LOADING SERVICE SUBSCRIPTIONS (source of truth)") print("=" * 70) rows = frappe.db.sql(""" SELECT name, customer, service_location, plan_name, monthly_price, status, service_category, billing_cycle, start_date, end_date, speed_down, speed_up, cancellation_date, cancellation_reason, legacy_service_id, product_sku, radius_user, radius_password FROM "tabService Subscription" ORDER BY customer, service_location, monthly_price DESC """, as_dict=True) by_customer = {} for r in rows: cust = r["customer"] if cust not in by_customer: by_customer[cust] = [] by_customer[cust].append(r) total = len(rows) active = sum(1 for r in rows if r["status"] == "Actif") customers = len(by_customer) print(f" Total: {total} records, {active} active, {customers} customers") return by_customer # ═══════════════════════════════════════════════════════════════ # STEP 2: Load all Subscriptions (ERPNext native) # ═══════════════════════════════════════════════════════════════ def load_subscriptions(): """Load all native Subscriptions grouped by party (customer).""" print("\n" + "=" * 70) print("LOADING SUBSCRIPTIONS (ERPNext native)") print("=" * 70) rows = frappe.db.sql(""" SELECT s.name, s.party as customer, s.status, s.start_date, s.end_date, s.actual_price, s.custom_description, s.item_code, s.item_group, s.billing_frequency, s.legacy_service_id, s.service_location, s.radius_user, s.cancel_at_period_end, s.cancelation_date, s.additional_discount_amount FROM "tabSubscription" s ORDER BY s.party, s.service_location, s.actual_price DESC """, as_dict=True) by_customer = {} by_legacy_id = {} for r in rows: cust = r["customer"] if cust not in by_customer: by_customer[cust] = [] by_customer[cust].append(r) if r.get("legacy_service_id"): by_legacy_id[r["legacy_service_id"]] = r total = len(rows) active = sum(1 for r in rows if r["status"] == "Active") customers = len(by_customer) print(f" Total: {total} records, {active} active, {customers} customers") return by_customer, by_legacy_id # ═══════════════════════════════════════════════════════════════ # STEP 3: Audit — Compare per-customer # ═══════════════════════════════════════════════════════════════ def audit(ss_by_cust, sub_by_cust, sub_by_legacy_id): """Compare per-customer and identify discrepancies.""" print("\n" + "=" * 70) print("AUDIT: COMPARING PER-CUSTOMER") print("=" * 70) all_customers = set(list(ss_by_cust.keys()) + list(sub_by_cust.keys())) # Discrepancy tracking missing_in_sub = [] # Service Subscriptions with no matching Subscription price_mismatches = [] # Matching records with different prices status_mismatches = [] # Matching records with different statuses count_mismatches = [] # Customers with different record counts total_mismatches = [] # Customers with different monthly totals customers_ok = 0 customers_mismatch = 0 for cust in sorted(all_customers): ss_list = ss_by_cust.get(cust, []) sub_list = sub_by_cust.get(cust, []) # Compare totals ss_total = sum(float(s.get("monthly_price") or 0) for s in ss_list if s["status"] == "Actif") sub_total = sum(float(s.get("actual_price") or 0) for s in sub_list if s["status"] == "Active") ss_active = [s for s in ss_list if s["status"] == "Actif"] sub_active = [s for s in sub_list if s["status"] == "Active"] has_issue = False # Count mismatch if len(ss_active) != len(sub_active): count_mismatches.append({ "customer": cust, "ss_count": len(ss_active), "sub_count": len(sub_active), "diff": len(ss_active) - len(sub_active), }) has_issue = True # Total mismatch (more than 0.01$ difference) if abs(ss_total - sub_total) > 0.01: total_mismatches.append({ "customer": cust, "ss_total": ss_total, "sub_total": sub_total, "diff": ss_total - sub_total, }) has_issue = True # Per-record matching via legacy_service_id for ss in ss_list: legacy_id = ss.get("legacy_service_id") if legacy_id and legacy_id in sub_by_legacy_id: sub = sub_by_legacy_id[legacy_id] # Price check ss_price = float(ss.get("monthly_price") or 0) sub_price = float(sub.get("actual_price") or 0) if abs(ss_price - sub_price) > 0.01: price_mismatches.append({ "customer": cust, "ss_name": ss["name"], "sub_name": sub["name"], "legacy_id": legacy_id, "plan": ss.get("plan_name") or "", "ss_price": ss_price, "sub_price": sub_price, "diff": ss_price - sub_price, }) has_issue = True # Status check ss_status = "Active" if ss["status"] == "Actif" else "Cancelled" sub_status = sub["status"] if ss_status != sub_status: status_mismatches.append({ "customer": cust, "ss_name": ss["name"], "sub_name": sub["name"], "legacy_id": legacy_id, "ss_status": ss["status"], "sub_status": sub_status, }) has_issue = True elif legacy_id: # No matching Subscription found missing_in_sub.append({ "customer": cust, "ss_name": ss["name"], "legacy_id": legacy_id, "plan_name": ss.get("plan_name") or "", "monthly_price": float(ss.get("monthly_price") or 0), "status": ss["status"], "service_location": ss.get("service_location") or "", "service_category": ss.get("service_category") or "", "billing_cycle": ss.get("billing_cycle") or "Mensuel", "start_date": ss.get("start_date"), "product_sku": ss.get("product_sku") or "", "radius_user": ss.get("radius_user") or "", "radius_password": ss.get("radius_password") or "", }) has_issue = True if has_issue: customers_mismatch += 1 else: customers_ok += 1 # ── Report ── print(f"\n Customers OK (totals match): {customers_ok}") print(f" Customers with discrepancies: {customers_mismatch}") print(f" ---") print(f" Missing in Subscription: {len(missing_in_sub)}") print(f" Price mismatches: {len(price_mismatches)}") print(f" Status mismatches: {len(status_mismatches)}") print(f" Count mismatches: {len(count_mismatches)}") print(f" Total (monthly $) mismatches: {len(total_mismatches)}") # Top 20 total mismatches if total_mismatches: print(f"\n TOP 20 MONTHLY TOTAL MISMATCHES:") sorted_tm = sorted(total_mismatches, key=lambda x: abs(x["diff"]), reverse=True) for tm in sorted_tm[:20]: print(f" {tm['customer']:25s} SS: {tm['ss_total']:>8.2f} SUB: {tm['sub_total']:>8.2f} DIFF: {tm['diff']:>+8.2f}") # Top 20 count mismatches if count_mismatches: print(f"\n TOP 20 COUNT MISMATCHES:") sorted_cm = sorted(count_mismatches, key=lambda x: abs(x["diff"]), reverse=True) for cm in sorted_cm[:20]: print(f" {cm['customer']:25s} SS: {cm['ss_count']:3d} SUB: {cm['sub_count']:3d} DIFF: {cm['diff']:+3d}") # Missing by category if missing_in_sub: cats = {} for m in missing_in_sub: cat = m["service_category"] or "Unknown" cats[cat] = cats.get(cat, 0) + 1 print(f"\n MISSING IN SUBSCRIPTION BY CATEGORY:") for cat, count in sorted(cats.items(), key=lambda x: -x[1]): print(f" {cat:25s} {count:5d}") return { "missing": missing_in_sub, "price_mismatches": price_mismatches, "status_mismatches": status_mismatches, "count_mismatches": count_mismatches, "total_mismatches": total_mismatches, } # ═══════════════════════════════════════════════════════════════ # STEP 4: Sync — Create missing Subscriptions # ═══════════════════════════════════════════════════════════════ def sync_missing(missing_records): """Create native Subscriptions for records missing from the Subscription table.""" print("\n" + "=" * 70) print(f"SYNC: CREATING {len(missing_records)} MISSING SUBSCRIPTIONS") print("=" * 70) if not missing_records: print(" Nothing to sync!") return ts = now() # Load existing subscription plans plans = frappe.db.sql('SELECT plan_name, name FROM "tabSubscription Plan"', as_dict=True) plan_map = {p["plan_name"]: p["name"] for p in plans} # Load existing items (to find or create plans) items = frappe.db.sql('SELECT name, item_name, item_group FROM "tabItem"', as_dict=True) item_map = {i["name"]: i for i in items} created = 0 errors = 0 plan_created = 0 for rec in missing_records: sku = rec["product_sku"] or "UNKNOWN" plan_name = f"PLAN-{sku}" # Create plan if it doesn't exist if plan_name not in plan_map: plan_id = uid("SP-") try: frappe.db.sql(""" INSERT INTO "tabSubscription Plan" ( name, creation, modified, modified_by, owner, docstatus, idx, plan_name, item, currency, price_determination, cost, billing_interval, billing_interval_count ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, 'CAD', 'Fixed Rate', %s, %s, 1) """, (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, abs(rec["monthly_price"]), "Year" if rec["billing_cycle"] == "Annuel" else "Month")) plan_map[plan_name] = plan_id plan_created += 1 except Exception as e: frappe.db.rollback() print(f" ERR plan {plan_name}: {str(e)[:80]}") # Determine status status = "Active" if rec["status"] == "Actif" else "Cancelled" # Determine discount: negative price = discount record, handle differently price = rec["monthly_price"] discount = 0 if price < 0: # This is a rebate/discount line — store as negative actual_price discount = 0 # Don't use additional_discount for negative-price lines # Start date start_date = rec.get("start_date") or "2020-01-01" if hasattr(start_date, 'strftime'): start_date = start_date.strftime("%Y-%m-%d") # Billing frequency billing_freq = "A" if rec["billing_cycle"] == "Annuel" else "M" sub_id = uid("SUB-") try: frappe.db.sql(""" INSERT INTO "tabSubscription" ( name, creation, modified, modified_by, owner, docstatus, idx, party_type, party, company, status, start_date, generate_invoice_at, days_until_due, follow_calendar_months, generate_new_invoices_past_due_date, submit_invoice, cancel_at_period_end, sales_tax_template, additional_discount_amount, radius_user, radius_pwd, legacy_service_id, service_location, actual_price, custom_description, item_code, item_group, billing_frequency ) VALUES ( %s, %s, %s, %s, %s, 0, 0, 'Customer', %s, %s, %s, %s, 'Beginning of the current subscription period', 30, 0, 1, 0, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) """, (sub_id, ts, ts, ADMIN, ADMIN, rec["customer"], COMPANY, status, start_date, TAX_TEMPLATE, discount, rec.get("radius_user") or None, rec.get("radius_password") or None, rec["legacy_id"], rec.get("service_location") or None, price, rec.get("plan_name") or "", sku, rec.get("service_category") or "", billing_freq)) # Create plan detail child record if plan_name in plan_map: frappe.db.sql(""" INSERT INTO "tabSubscription Plan Detail" ( name, creation, modified, modified_by, owner, docstatus, idx, plan, qty, parent, parentfield, parenttype ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'plans', 'Subscription') """, (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id)) created += 1 if created % 500 == 0: frappe.db.commit() print(f" [{created}/{len(missing_records)}] created...") except Exception as e: errors += 1 frappe.db.rollback() if errors <= 20: print(f" ERR {rec['customer']} {rec['ss_name']}: {str(e)[:100]}") frappe.db.commit() print(f"\n Plans created: {plan_created}") print(f" Subscriptions created: {created}") print(f" Errors: {errors}") # ═══════════════════════════════════════════════════════════════ # STEP 5: Fix — Correct price/status mismatches # ═══════════════════════════════════════════════════════════════ def fix_mismatches(price_mismatches, status_mismatches): """Fix price and status mismatches on existing Subscriptions.""" print("\n" + "=" * 70) print(f"FIX: CORRECTING {len(price_mismatches)} PRICE + {len(status_mismatches)} STATUS MISMATCHES") print("=" * 70) fixed_price = 0 fixed_status = 0 for pm in price_mismatches: try: frappe.db.sql(""" UPDATE "tabSubscription" SET actual_price = %s, modified = %s WHERE name = %s """, (pm["ss_price"], now(), pm["sub_name"])) fixed_price += 1 except Exception as e: print(f" ERR price fix {pm['sub_name']}: {str(e)[:80]}") for sm in status_mismatches: new_status = "Active" if sm["ss_status"] == "Actif" else "Cancelled" try: frappe.db.sql(""" UPDATE "tabSubscription" SET status = %s, modified = %s WHERE name = %s """, (new_status, now(), sm["sub_name"])) fixed_status += 1 except Exception as e: print(f" ERR status fix {sm['sub_name']}: {str(e)[:80]}") frappe.db.commit() print(f" Price fixes: {fixed_price}") print(f" Status fixes: {fixed_status}") # ═══════════════════════════════════════════════════════════════ # MAIN # ═══════════════════════════════════════════════════════════════ def main(): mode = sys.argv[1] if len(sys.argv) > 1 else "--audit" print(f"\nMode: {mode}") print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") # Load data ss_by_cust = load_service_subscriptions() sub_by_cust, sub_by_legacy_id = load_subscriptions() # Audit results = audit(ss_by_cust, sub_by_cust, sub_by_legacy_id) if mode in ("--sync", "--full"): sync_missing(results["missing"]) if mode in ("--fix", "--full"): fix_mismatches(results["price_mismatches"], results["status_mismatches"]) # Write report file report_path = "/tmp/reconcile_report.json" report = { "timestamp": datetime.now().isoformat(), "mode": mode, "summary": { "missing_in_subscription": len(results["missing"]), "price_mismatches": len(results["price_mismatches"]), "status_mismatches": len(results["status_mismatches"]), "count_mismatches": len(results["count_mismatches"]), "total_mismatches": len(results["total_mismatches"]), }, "total_mismatches_top50": sorted(results["total_mismatches"], key=lambda x: abs(x["diff"]), reverse=True)[:50], "missing_sample": results["missing"][:50], } with open(report_path, "w") as f: json.dump(report, f, indent=2, default=str) print(f"\nReport saved: {report_path}") frappe.clear_cache() print("\nDone — cache cleared") if __name__ == "__main__": main()