4a8718f67c
- Switch Ops data source from Subscription to Service Subscription (source of truth)
- Reimport 39,630 native Subscriptions from Service Subscription data
- Rename 15,302 customers to CUST-{legacy_customer_id} (eliminates hex UUIDs)
- Rename all doctypes to zero-padded 10-digit numeric format:
SINV-0000001234, PE-0000001234, ISS-0000001234, LOC-0000001234,
EQP-0000001234, SUB-0000001234, ASUB-0000001234
- Fix subscription pricing: LPB4 now correctly shows 0$/month
- Update ASUB- prefix detection in useSubscriptionActions.js
- Add reconciliation, reimport, and rename migration scripts
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
492 lines
20 KiB
Python
492 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Reconcile Service Subscription (complete legacy import) vs Subscription (ERPNext native).
|
|
|
|
Identifies per-customer discrepancies:
|
|
- Missing records in Subscription that exist in Service Subscription
|
|
- Price/status mismatches
|
|
- Generates a sync plan to fix the native Subscription table
|
|
|
|
Run inside erpnext-backend-1:
|
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reconcile_subscriptions.py
|
|
|
|
Modes:
|
|
--audit Audit only — report discrepancies (default)
|
|
--sync Create missing Subscriptions from Service Subscriptions
|
|
--fix Fix price/status mismatches on existing Subscriptions
|
|
--full Audit + Sync + Fix (do everything)
|
|
"""
|
|
import sys
|
|
import os
|
|
import json
|
|
from datetime import datetime, timezone
|
|
import uuid
|
|
|
|
os.chdir("/home/frappe/frappe-bench/sites")
|
|
|
|
import frappe
|
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
|
frappe.connect()
|
|
frappe.local.flags.ignore_permissions = True
|
|
print(f"Connected: {frappe.local.site}")
|
|
|
|
ADMIN = "Administrator"
|
|
COMPANY = "TARGO"
|
|
TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T"
|
|
|
|
|
|
def uid(prefix=""):
|
|
return prefix + uuid.uuid4().hex[:10]
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# STEP 1: Load all Service Subscriptions (source of truth)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def load_service_subscriptions():
|
|
"""Load all Service Subscriptions grouped by customer."""
|
|
print("\n" + "=" * 70)
|
|
print("LOADING SERVICE SUBSCRIPTIONS (source of truth)")
|
|
print("=" * 70)
|
|
|
|
rows = frappe.db.sql("""
|
|
SELECT name, customer, service_location, plan_name, monthly_price,
|
|
status, service_category, billing_cycle, start_date, end_date,
|
|
speed_down, speed_up, cancellation_date, cancellation_reason,
|
|
legacy_service_id, product_sku, radius_user, radius_password
|
|
FROM "tabService Subscription"
|
|
ORDER BY customer, service_location, monthly_price DESC
|
|
""", as_dict=True)
|
|
|
|
by_customer = {}
|
|
for r in rows:
|
|
cust = r["customer"]
|
|
if cust not in by_customer:
|
|
by_customer[cust] = []
|
|
by_customer[cust].append(r)
|
|
|
|
total = len(rows)
|
|
active = sum(1 for r in rows if r["status"] == "Actif")
|
|
customers = len(by_customer)
|
|
print(f" Total: {total} records, {active} active, {customers} customers")
|
|
return by_customer
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# STEP 2: Load all Subscriptions (ERPNext native)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def load_subscriptions():
|
|
"""Load all native Subscriptions grouped by party (customer)."""
|
|
print("\n" + "=" * 70)
|
|
print("LOADING SUBSCRIPTIONS (ERPNext native)")
|
|
print("=" * 70)
|
|
|
|
rows = frappe.db.sql("""
|
|
SELECT s.name, s.party as customer, s.status, s.start_date, s.end_date,
|
|
s.actual_price, s.custom_description, s.item_code, s.item_group,
|
|
s.billing_frequency, s.legacy_service_id, s.service_location,
|
|
s.radius_user, s.cancel_at_period_end, s.cancelation_date,
|
|
s.additional_discount_amount
|
|
FROM "tabSubscription" s
|
|
ORDER BY s.party, s.service_location, s.actual_price DESC
|
|
""", as_dict=True)
|
|
|
|
by_customer = {}
|
|
by_legacy_id = {}
|
|
for r in rows:
|
|
cust = r["customer"]
|
|
if cust not in by_customer:
|
|
by_customer[cust] = []
|
|
by_customer[cust].append(r)
|
|
if r.get("legacy_service_id"):
|
|
by_legacy_id[r["legacy_service_id"]] = r
|
|
|
|
total = len(rows)
|
|
active = sum(1 for r in rows if r["status"] == "Active")
|
|
customers = len(by_customer)
|
|
print(f" Total: {total} records, {active} active, {customers} customers")
|
|
return by_customer, by_legacy_id
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# STEP 3: Audit — Compare per-customer
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def audit(ss_by_cust, sub_by_cust, sub_by_legacy_id):
|
|
"""Compare per-customer and identify discrepancies."""
|
|
print("\n" + "=" * 70)
|
|
print("AUDIT: COMPARING PER-CUSTOMER")
|
|
print("=" * 70)
|
|
|
|
all_customers = set(list(ss_by_cust.keys()) + list(sub_by_cust.keys()))
|
|
|
|
# Discrepancy tracking
|
|
missing_in_sub = [] # Service Subscriptions with no matching Subscription
|
|
price_mismatches = [] # Matching records with different prices
|
|
status_mismatches = [] # Matching records with different statuses
|
|
count_mismatches = [] # Customers with different record counts
|
|
total_mismatches = [] # Customers with different monthly totals
|
|
|
|
customers_ok = 0
|
|
customers_mismatch = 0
|
|
|
|
for cust in sorted(all_customers):
|
|
ss_list = ss_by_cust.get(cust, [])
|
|
sub_list = sub_by_cust.get(cust, [])
|
|
|
|
# Compare totals
|
|
ss_total = sum(float(s.get("monthly_price") or 0) for s in ss_list if s["status"] == "Actif")
|
|
sub_total = sum(float(s.get("actual_price") or 0) for s in sub_list if s["status"] == "Active")
|
|
|
|
ss_active = [s for s in ss_list if s["status"] == "Actif"]
|
|
sub_active = [s for s in sub_list if s["status"] == "Active"]
|
|
|
|
has_issue = False
|
|
|
|
# Count mismatch
|
|
if len(ss_active) != len(sub_active):
|
|
count_mismatches.append({
|
|
"customer": cust,
|
|
"ss_count": len(ss_active),
|
|
"sub_count": len(sub_active),
|
|
"diff": len(ss_active) - len(sub_active),
|
|
})
|
|
has_issue = True
|
|
|
|
# Total mismatch (more than 0.01$ difference)
|
|
if abs(ss_total - sub_total) > 0.01:
|
|
total_mismatches.append({
|
|
"customer": cust,
|
|
"ss_total": ss_total,
|
|
"sub_total": sub_total,
|
|
"diff": ss_total - sub_total,
|
|
})
|
|
has_issue = True
|
|
|
|
# Per-record matching via legacy_service_id
|
|
for ss in ss_list:
|
|
legacy_id = ss.get("legacy_service_id")
|
|
if legacy_id and legacy_id in sub_by_legacy_id:
|
|
sub = sub_by_legacy_id[legacy_id]
|
|
# Price check
|
|
ss_price = float(ss.get("monthly_price") or 0)
|
|
sub_price = float(sub.get("actual_price") or 0)
|
|
if abs(ss_price - sub_price) > 0.01:
|
|
price_mismatches.append({
|
|
"customer": cust,
|
|
"ss_name": ss["name"],
|
|
"sub_name": sub["name"],
|
|
"legacy_id": legacy_id,
|
|
"plan": ss.get("plan_name") or "",
|
|
"ss_price": ss_price,
|
|
"sub_price": sub_price,
|
|
"diff": ss_price - sub_price,
|
|
})
|
|
has_issue = True
|
|
|
|
# Status check
|
|
ss_status = "Active" if ss["status"] == "Actif" else "Cancelled"
|
|
sub_status = sub["status"]
|
|
if ss_status != sub_status:
|
|
status_mismatches.append({
|
|
"customer": cust,
|
|
"ss_name": ss["name"],
|
|
"sub_name": sub["name"],
|
|
"legacy_id": legacy_id,
|
|
"ss_status": ss["status"],
|
|
"sub_status": sub_status,
|
|
})
|
|
has_issue = True
|
|
elif legacy_id:
|
|
# No matching Subscription found
|
|
missing_in_sub.append({
|
|
"customer": cust,
|
|
"ss_name": ss["name"],
|
|
"legacy_id": legacy_id,
|
|
"plan_name": ss.get("plan_name") or "",
|
|
"monthly_price": float(ss.get("monthly_price") or 0),
|
|
"status": ss["status"],
|
|
"service_location": ss.get("service_location") or "",
|
|
"service_category": ss.get("service_category") or "",
|
|
"billing_cycle": ss.get("billing_cycle") or "Mensuel",
|
|
"start_date": ss.get("start_date"),
|
|
"product_sku": ss.get("product_sku") or "",
|
|
"radius_user": ss.get("radius_user") or "",
|
|
"radius_password": ss.get("radius_password") or "",
|
|
})
|
|
has_issue = True
|
|
|
|
if has_issue:
|
|
customers_mismatch += 1
|
|
else:
|
|
customers_ok += 1
|
|
|
|
# ── Report ──
|
|
print(f"\n Customers OK (totals match): {customers_ok}")
|
|
print(f" Customers with discrepancies: {customers_mismatch}")
|
|
print(f" ---")
|
|
print(f" Missing in Subscription: {len(missing_in_sub)}")
|
|
print(f" Price mismatches: {len(price_mismatches)}")
|
|
print(f" Status mismatches: {len(status_mismatches)}")
|
|
print(f" Count mismatches: {len(count_mismatches)}")
|
|
print(f" Total (monthly $) mismatches: {len(total_mismatches)}")
|
|
|
|
# Top 20 total mismatches
|
|
if total_mismatches:
|
|
print(f"\n TOP 20 MONTHLY TOTAL MISMATCHES:")
|
|
sorted_tm = sorted(total_mismatches, key=lambda x: abs(x["diff"]), reverse=True)
|
|
for tm in sorted_tm[:20]:
|
|
print(f" {tm['customer']:25s} SS: {tm['ss_total']:>8.2f} SUB: {tm['sub_total']:>8.2f} DIFF: {tm['diff']:>+8.2f}")
|
|
|
|
# Top 20 count mismatches
|
|
if count_mismatches:
|
|
print(f"\n TOP 20 COUNT MISMATCHES:")
|
|
sorted_cm = sorted(count_mismatches, key=lambda x: abs(x["diff"]), reverse=True)
|
|
for cm in sorted_cm[:20]:
|
|
print(f" {cm['customer']:25s} SS: {cm['ss_count']:3d} SUB: {cm['sub_count']:3d} DIFF: {cm['diff']:+3d}")
|
|
|
|
# Missing by category
|
|
if missing_in_sub:
|
|
cats = {}
|
|
for m in missing_in_sub:
|
|
cat = m["service_category"] or "Unknown"
|
|
cats[cat] = cats.get(cat, 0) + 1
|
|
print(f"\n MISSING IN SUBSCRIPTION BY CATEGORY:")
|
|
for cat, count in sorted(cats.items(), key=lambda x: -x[1]):
|
|
print(f" {cat:25s} {count:5d}")
|
|
|
|
return {
|
|
"missing": missing_in_sub,
|
|
"price_mismatches": price_mismatches,
|
|
"status_mismatches": status_mismatches,
|
|
"count_mismatches": count_mismatches,
|
|
"total_mismatches": total_mismatches,
|
|
}
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# STEP 4: Sync — Create missing Subscriptions
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def sync_missing(missing_records):
|
|
"""Create native Subscriptions for records missing from the Subscription table."""
|
|
print("\n" + "=" * 70)
|
|
print(f"SYNC: CREATING {len(missing_records)} MISSING SUBSCRIPTIONS")
|
|
print("=" * 70)
|
|
|
|
if not missing_records:
|
|
print(" Nothing to sync!")
|
|
return
|
|
|
|
ts = now()
|
|
|
|
# Load existing subscription plans
|
|
plans = frappe.db.sql('SELECT plan_name, name FROM "tabSubscription Plan"', as_dict=True)
|
|
plan_map = {p["plan_name"]: p["name"] for p in plans}
|
|
|
|
# Load existing items (to find or create plans)
|
|
items = frappe.db.sql('SELECT name, item_name, item_group FROM "tabItem"', as_dict=True)
|
|
item_map = {i["name"]: i for i in items}
|
|
|
|
created = 0
|
|
errors = 0
|
|
plan_created = 0
|
|
|
|
for rec in missing_records:
|
|
sku = rec["product_sku"] or "UNKNOWN"
|
|
plan_name = f"PLAN-{sku}"
|
|
|
|
# Create plan if it doesn't exist
|
|
if plan_name not in plan_map:
|
|
plan_id = uid("SP-")
|
|
try:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSubscription Plan" (
|
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
|
plan_name, item, currency, price_determination, cost,
|
|
billing_interval, billing_interval_count
|
|
) VALUES (%s, %s, %s, %s, %s, 0, 0,
|
|
%s, %s, 'CAD', 'Fixed Rate', %s, %s, 1)
|
|
""", (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku,
|
|
abs(rec["monthly_price"]),
|
|
"Year" if rec["billing_cycle"] == "Annuel" else "Month"))
|
|
plan_map[plan_name] = plan_id
|
|
plan_created += 1
|
|
except Exception as e:
|
|
frappe.db.rollback()
|
|
print(f" ERR plan {plan_name}: {str(e)[:80]}")
|
|
|
|
# Determine status
|
|
status = "Active" if rec["status"] == "Actif" else "Cancelled"
|
|
|
|
# Determine discount: negative price = discount record, handle differently
|
|
price = rec["monthly_price"]
|
|
discount = 0
|
|
if price < 0:
|
|
# This is a rebate/discount line — store as negative actual_price
|
|
discount = 0 # Don't use additional_discount for negative-price lines
|
|
|
|
# Start date
|
|
start_date = rec.get("start_date") or "2020-01-01"
|
|
if hasattr(start_date, 'strftime'):
|
|
start_date = start_date.strftime("%Y-%m-%d")
|
|
|
|
# Billing frequency
|
|
billing_freq = "A" if rec["billing_cycle"] == "Annuel" else "M"
|
|
|
|
sub_id = uid("SUB-")
|
|
try:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSubscription" (
|
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
|
party_type, party, company, status,
|
|
start_date, generate_invoice_at, days_until_due,
|
|
follow_calendar_months, generate_new_invoices_past_due_date,
|
|
submit_invoice, cancel_at_period_end,
|
|
sales_tax_template,
|
|
additional_discount_amount,
|
|
radius_user, radius_pwd, legacy_service_id,
|
|
service_location,
|
|
actual_price, custom_description, item_code, item_group,
|
|
billing_frequency
|
|
) VALUES (
|
|
%s, %s, %s, %s, %s, 0, 0,
|
|
'Customer', %s, %s, %s,
|
|
%s, 'Beginning of the current subscription period', 30,
|
|
0, 1, 0, 0,
|
|
%s, %s,
|
|
%s, %s, %s,
|
|
%s,
|
|
%s, %s, %s, %s, %s
|
|
)
|
|
""", (sub_id, ts, ts, ADMIN, ADMIN,
|
|
rec["customer"], COMPANY, status,
|
|
start_date,
|
|
TAX_TEMPLATE, discount,
|
|
rec.get("radius_user") or None,
|
|
rec.get("radius_password") or None,
|
|
rec["legacy_id"],
|
|
rec.get("service_location") or None,
|
|
price, rec.get("plan_name") or "",
|
|
sku, rec.get("service_category") or "",
|
|
billing_freq))
|
|
|
|
# Create plan detail child record
|
|
if plan_name in plan_map:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabSubscription Plan Detail" (
|
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
|
plan, qty,
|
|
parent, parentfield, parenttype
|
|
) VALUES (%s, %s, %s, %s, %s, 0, 1,
|
|
%s, 1,
|
|
%s, 'plans', 'Subscription')
|
|
""", (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id))
|
|
|
|
created += 1
|
|
|
|
if created % 500 == 0:
|
|
frappe.db.commit()
|
|
print(f" [{created}/{len(missing_records)}] created...")
|
|
|
|
except Exception as e:
|
|
errors += 1
|
|
frappe.db.rollback()
|
|
if errors <= 20:
|
|
print(f" ERR {rec['customer']} {rec['ss_name']}: {str(e)[:100]}")
|
|
|
|
frappe.db.commit()
|
|
print(f"\n Plans created: {plan_created}")
|
|
print(f" Subscriptions created: {created}")
|
|
print(f" Errors: {errors}")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# STEP 5: Fix — Correct price/status mismatches
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def fix_mismatches(price_mismatches, status_mismatches):
|
|
"""Fix price and status mismatches on existing Subscriptions."""
|
|
print("\n" + "=" * 70)
|
|
print(f"FIX: CORRECTING {len(price_mismatches)} PRICE + {len(status_mismatches)} STATUS MISMATCHES")
|
|
print("=" * 70)
|
|
|
|
fixed_price = 0
|
|
fixed_status = 0
|
|
|
|
for pm in price_mismatches:
|
|
try:
|
|
frappe.db.sql("""
|
|
UPDATE "tabSubscription"
|
|
SET actual_price = %s, modified = %s
|
|
WHERE name = %s
|
|
""", (pm["ss_price"], now(), pm["sub_name"]))
|
|
fixed_price += 1
|
|
except Exception as e:
|
|
print(f" ERR price fix {pm['sub_name']}: {str(e)[:80]}")
|
|
|
|
for sm in status_mismatches:
|
|
new_status = "Active" if sm["ss_status"] == "Actif" else "Cancelled"
|
|
try:
|
|
frappe.db.sql("""
|
|
UPDATE "tabSubscription"
|
|
SET status = %s, modified = %s
|
|
WHERE name = %s
|
|
""", (new_status, now(), sm["sub_name"]))
|
|
fixed_status += 1
|
|
except Exception as e:
|
|
print(f" ERR status fix {sm['sub_name']}: {str(e)[:80]}")
|
|
|
|
frappe.db.commit()
|
|
print(f" Price fixes: {fixed_price}")
|
|
print(f" Status fixes: {fixed_status}")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# MAIN
|
|
# ═══════════════════════════════════════════════════════════════
|
|
def main():
|
|
mode = sys.argv[1] if len(sys.argv) > 1 else "--audit"
|
|
|
|
print(f"\nMode: {mode}")
|
|
print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
|
|
# Load data
|
|
ss_by_cust = load_service_subscriptions()
|
|
sub_by_cust, sub_by_legacy_id = load_subscriptions()
|
|
|
|
# Audit
|
|
results = audit(ss_by_cust, sub_by_cust, sub_by_legacy_id)
|
|
|
|
if mode in ("--sync", "--full"):
|
|
sync_missing(results["missing"])
|
|
|
|
if mode in ("--fix", "--full"):
|
|
fix_mismatches(results["price_mismatches"], results["status_mismatches"])
|
|
|
|
# Write report file
|
|
report_path = "/tmp/reconcile_report.json"
|
|
report = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"mode": mode,
|
|
"summary": {
|
|
"missing_in_subscription": len(results["missing"]),
|
|
"price_mismatches": len(results["price_mismatches"]),
|
|
"status_mismatches": len(results["status_mismatches"]),
|
|
"count_mismatches": len(results["count_mismatches"]),
|
|
"total_mismatches": len(results["total_mismatches"]),
|
|
},
|
|
"total_mismatches_top50": sorted(results["total_mismatches"], key=lambda x: abs(x["diff"]), reverse=True)[:50],
|
|
"missing_sample": results["missing"][:50],
|
|
}
|
|
with open(report_path, "w") as f:
|
|
json.dump(report, f, indent=2, default=str)
|
|
print(f"\nReport saved: {report_path}")
|
|
|
|
frappe.clear_cache()
|
|
print("\nDone — cache cleared")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|