gigafibre-fsm/scripts/migration/reimport_subscriptions.py

#!/usr/bin/env python3
"""
Clean reimport of native Subscriptions from Service Subscription (source of truth).

Fixes the migration issue where Phase 3/4 created hex customer IDs (CUST-04ea550dcf)
instead of mapping to the correct sequential IDs (CUST-988).

Steps:
  1. Purge all native Subscriptions + Plan Details
  2. Recreate Subscriptions from Service Subscription data (correct customer mapping)
  3. Delete orphaned hex customers (no remaining FK references)
  4. Verify totals match

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reimport_subscriptions.py [--dry-run]

Log: stdout (run with tee)
"""
import sys
import os
import uuid
import time
import html
from datetime import datetime, timezone

os.chdir("/home/frappe/frappe-bench/sites")
import frappe

frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print(f"Connected: {frappe.local.site}")

DRY_RUN = "--dry-run" in sys.argv
if DRY_RUN:
    print("*** DRY RUN — no changes will be written ***")

ADMIN = "Administrator"
COMPANY = "TARGO"
TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T"


def uid(prefix=""):
    return prefix + uuid.uuid4().hex[:10]


def now():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")


# ═══════════════════════════════════════════════════════════════
# STEP 1: PURGE native Subscriptions
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 1: PURGE EXISTING SUBSCRIPTIONS")
print("=" * 70)

sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0]
spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0]
sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0]

print(f"  Current counts: {sub_count} Subscriptions, {spd_count} Plan Details, {sp_count} Plans")

if not DRY_RUN:
    frappe.db.sql('DELETE FROM "tabSubscription Plan Detail"')
    frappe.db.sql('DELETE FROM "tabSubscription"')
    frappe.db.sql('DELETE FROM "tabSubscription Plan"')
    frappe.db.commit()
    print("  Purged all Subscriptions, Plan Details, and Plans.")
else:
    print("  [DRY RUN] Would purge all Subscriptions, Plan Details, and Plans.")


# ═══════════════════════════════════════════════════════════════
# STEP 2: LOAD Service Subscriptions (source of truth)
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 2: LOAD SERVICE SUBSCRIPTIONS")
print("=" * 70)

ss_rows = frappe.db.sql("""
    SELECT name, customer, service_location, plan_name, monthly_price,
           status, service_category, billing_cycle, start_date, end_date,
           speed_down, speed_up, cancellation_date, cancellation_reason,
           legacy_service_id, product_sku, radius_user, radius_password, notes
    FROM "tabService Subscription"
    ORDER BY customer, service_location, monthly_price DESC
""", as_dict=True)

total = len(ss_rows)
active = sum(1 for r in ss_rows if r["status"] == "Actif")
customers = len(set(r["customer"] for r in ss_rows))
print(f"  Loaded {total} Service Subscriptions ({active} active) for {customers} customers")


# ═══════════════════════════════════════════════════════════════
# STEP 3: BUILD Subscription Plans from unique SKUs
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 3: CREATE SUBSCRIPTION PLANS")
print("=" * 70)
t0 = time.time()

# Collect unique SKUs with their typical price and category
sku_info = {}
for r in ss_rows:
    sku = r.get("product_sku") or ""
    if not sku:
        continue
    if sku not in sku_info:
        sku_info[sku] = {
            "price": abs(float(r["monthly_price"] or 0)),
            "category": r.get("service_category") or "",
            "billing_cycle": r.get("billing_cycle") or "Mensuel",
            "plan_name": r.get("plan_name") or sku,
        }

# Also add a fallback plan for records without SKU
plan_map = {}  # plan_name → plan_id
plans_created = 0

if not DRY_RUN:
    ts = now()
    for sku, info in sku_info.items():
        plan_name = f"PLAN-{sku}"
        plan_id = uid("SP-")
        billing_interval = "Year" if info["billing_cycle"] == "Annuel" else "Month"

        try:
            frappe.db.sql("""
                INSERT INTO "tabSubscription Plan" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    plan_name, item, currency, price_determination, cost,
                    billing_interval, billing_interval_count
                ) VALUES (%s, %s, %s, %s, %s, 0, 0,
                          %s, %s, 'CAD', 'Fixed Rate', %s, %s, 1)
            """, (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, info["price"], billing_interval))
            plan_map[plan_name] = plan_id
            plans_created += 1
        except Exception as e:
            frappe.db.rollback()
            print(f"  ERR plan {sku}: {str(e)[:80]}")

    frappe.db.commit()

print(f"  Created {plans_created} Subscription Plans [{time.time()-t0:.1f}s]")


# ═══════════════════════════════════════════════════════════════
# STEP 4: CREATE Subscriptions from Service Subscriptions
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 4: CREATE SUBSCRIPTIONS")
print("=" * 70)
t0 = time.time()

# Verify all customers exist
all_customers = set(r["customer"] for r in ss_rows)
existing_customers = set(
    r[0] for r in frappe.db.sql('SELECT name FROM "tabCustomer"')
)
missing_customers = all_customers - existing_customers
if missing_customers:
    print(f"  WARNING: {len(missing_customers)} customers referenced but don't exist!")
    for c in list(missing_customers)[:10]:
        print(f"    {c}")

sub_ok = sub_err = 0
ts = now()

for i, ss in enumerate(ss_rows):
    if DRY_RUN:
        sub_ok += 1
        continue

    cust = ss["customer"]
    if cust in missing_customers:
        sub_err += 1
        continue

    # Map status
    if ss["status"] == "Actif":
        status = "Active"
    elif ss["status"] == "Annulé":
        status = "Cancelled"
    elif ss["status"] == "Suspendu":
        status = "Past Due Date"
    else:
        status = "Active"  # default

    # Price
    price = float(ss["monthly_price"] or 0)

    # Start date
    start_date = ss.get("start_date") or "2020-01-01"
    if hasattr(start_date, "strftime"):
        start_date = start_date.strftime("%Y-%m-%d")

    # End/cancellation date
    end_date = ss.get("end_date")
    if hasattr(end_date, "strftime"):
        end_date = end_date.strftime("%Y-%m-%d")
    cancel_date = ss.get("cancellation_date")
    if hasattr(cancel_date, "strftime"):
        cancel_date = cancel_date.strftime("%Y-%m-%d")

    # Billing frequency
    billing_freq = "A" if ss.get("billing_cycle") == "Annuel" else "M"

    # SKU / plan
    sku = ss.get("product_sku") or ""
    plan_name = f"PLAN-{sku}" if sku else ""

    # Category
    category = ss.get("service_category") or ""

    # Description: plan_name from SS (unescape HTML entities from legacy data)
    description = html.unescape(ss.get("plan_name") or "")

    sub_id = uid("SUB-")

    try:
        frappe.db.sql("""
            INSERT INTO "tabSubscription" (
                name, creation, modified, modified_by, owner, docstatus, idx,
                party_type, party, company, status,
                start_date, generate_invoice_at, days_until_due,
                follow_calendar_months, generate_new_invoices_past_due_date,
                submit_invoice, cancel_at_period_end,
                sales_tax_template,
                additional_discount_amount,
                radius_user, radius_pwd, legacy_service_id,
                service_location,
                actual_price, custom_description, item_code, item_group,
                billing_frequency,
                cancelation_date
            ) VALUES (
                %s, %s, %s, %s, %s, 0, 0,
                'Customer', %s, %s, %s,
                %s, 'Beginning of the current subscription period', 30,
                1, 1, 0, 0,
                %s, 0,
                %s, %s, %s,
                %s,
                %s, %s, %s, %s, %s,
                %s
            )
        """, (sub_id, ts, ts, ADMIN, ADMIN,
              cust, COMPANY, status,
              start_date,
              TAX_TEMPLATE,
              ss.get("radius_user") or None,
              ss.get("radius_password") or None,
              ss.get("legacy_service_id"),
              ss.get("service_location") or None,
              price, description, sku, category, billing_freq,
              cancel_date))

        # Create plan detail child record
        if plan_name and plan_name in plan_map:
            frappe.db.sql("""
                INSERT INTO "tabSubscription Plan Detail" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    plan, qty,
                    parent, parentfield, parenttype
                ) VALUES (%s, %s, %s, %s, %s, 0, 1,
                          %s, 1,
                          %s, 'plans', 'Subscription')
            """, (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id))

        sub_ok += 1

        if sub_ok % 2000 == 0:
            frappe.db.commit()
            print(f"  [{sub_ok}/{total}] created... [{time.time()-t0:.0f}s]")

    except Exception as e:
        sub_err += 1
        frappe.db.rollback()
        if sub_err <= 20:
            print(f"  ERR {cust}/{ss['name']}: {str(e)[:100]}")

if not DRY_RUN:
    frappe.db.commit()

print(f"  Created {sub_ok} Subscriptions, {sub_err} errors [{time.time()-t0:.1f}s]")


# ═══════════════════════════════════════════════════════════════
# STEP 5: CLEANUP — Delete orphaned hex customers
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 5: CLEANUP ORPHANED HEX CUSTOMERS")
print("=" * 70)
t0 = time.time()

# Find hex-pattern customers (non-numeric suffix after CUST-)
hex_customers = frappe.db.sql("""
    SELECT name, customer_name, legacy_account_id
    FROM "tabCustomer"
    WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
""", as_dict=True)

print(f"  Found {len(hex_customers)} hex-pattern customers")

if hex_customers:
    # Check which hex customers still have FK references using subqueries (avoid ANY() limits)
    hex_subquery = """(SELECT name FROM "tabCustomer" WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$')"""

    fk_tables = [
        ("tabSales Invoice", "customer"),
        ("tabPayment Entry", "party"),
        ("tabIssue", "customer"),
        ("tabService Location", "customer"),
        ("tabService Subscription", "customer"),
        ("tabSubscription", "party"),
        ("tabService Equipment", "customer"),
    ]

    referenced = set()
    for table, col in fk_tables:
        try:
            refs = frappe.db.sql(
                f'SELECT DISTINCT "{col}" FROM "{table}" WHERE "{col}" IN {hex_subquery}'
            )
            for r in refs:
                referenced.add(r[0])
            print(f"    {table:30s} {col:15s} → {len(refs)} hex refs")
        except Exception as e:
            frappe.db.rollback()
            print(f"    ERR {table}: {str(e)[:60]}")

    orphaned = [c for c in hex_customers if c["name"] not in referenced]
    still_used = [c for c in hex_customers if c["name"] in referenced]

    print(f"  Still referenced (keep): {len(still_used)}")
    print(f"  Orphaned (can delete):   {len(orphaned)}")

    if orphaned and not DRY_RUN:
        # Delete orphaned hex customers using subquery (avoid ANY() limits)
        # First, create a temp table of referenced hex customers
        frappe.db.sql("""
            DELETE FROM "tabDynamic Link"
            WHERE link_doctype = 'Customer'
            AND link_name IN (SELECT name FROM "tabCustomer"
                WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
                AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL)
                AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL)
            )
        """)
        frappe.db.commit()

        deleted = frappe.db.sql("""
            DELETE FROM "tabCustomer"
            WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
            AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL)
            AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL)
        """)
        frappe.db.commit()
        print(f"  Deleted orphaned hex customers")
    elif orphaned:
        print(f"  [DRY RUN] Would delete {len(orphaned)} orphaned hex customers")
else:
    print("  No hex-pattern customers found.")


# ═══════════════════════════════════════════════════════════════
# STEP 6: VERIFY
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("STEP 6: VERIFICATION")
print("=" * 70)

new_sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0]
new_spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0]
new_sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0]
cust_count = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer"')[0][0]

print(f"  Subscriptions:     {new_sub_count}")
print(f"  Plan Details:      {new_spd_count}")
print(f"  Plans:             {new_sp_count}")
print(f"  Customers:         {cust_count}")

# Check for customer ID consistency: all subscriptions should point to sequential customers
hex_refs = frappe.db.sql("""
    SELECT COUNT(*) FROM "tabSubscription"
    WHERE party ~ '^CUST-' AND party !~ '^CUST-[0-9]+$'
""")[0][0]
print(f"  Subscriptions pointing to hex customers: {hex_refs} (should be 0)")

# Spot-check: LPB4 customer (legacy_account_id = 4)
lpb4 = frappe.db.sql("""
    SELECT name FROM "tabCustomer" WHERE legacy_account_id = 4
""", as_dict=True)
if lpb4:
    cust_name = lpb4[0]["name"]
    lpb4_subs = frappe.db.sql("""
        SELECT actual_price, item_code, custom_description, status, billing_frequency
        FROM "tabSubscription"
        WHERE party = %s
        ORDER BY actual_price DESC
    """, (cust_name,), as_dict=True)
    total_price = sum(float(s["actual_price"] or 0) for s in lpb4_subs if s["status"] == "Active")
    print(f"\n  Spot-check: {cust_name} (LPB4)")
    for s in lpb4_subs:
        print(f"    {s['item_code'] or '':15s}  {float(s['actual_price'] or 0):>8.2f}  {s['billing_frequency']}  {s['status']}  {s['custom_description'] or ''}")
    print(f"    {'TOTAL':15s}  {total_price:>8.2f}$/mo")

# Summary
print("\n" + "=" * 70)
print("COMPLETE")
print("=" * 70)
print(f"  Next: bench --site erp.gigafibre.ca clear-cache")
print()