gigafibre-fsm/scripts/migration/import_services_and_enrich_customers.py

"""
Import active services as Service Subscriptions and enrich Customer records
with full account details (phone, email, stripe, PPA, notes).

Also adds custom fields to Service Subscription for RADIUS/legacy data.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_services_and_enrich_customers.py
"""
import html
import frappe
import pymysql
import os
import time
import hashlib
from datetime import datetime
from decimal import Decimal

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

T_TOTAL = time.time()
now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")

conn = pymysql.connect(
    host="legacy-db",
    user="facturation",
    password="*******",
    database="gestionclient",
    cursorclass=pymysql.cursors.DictCursor
)

# ═══════════════════════════════════════════════════════════════
# PHASE 1: Add custom fields to Service Subscription
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 1: ADD CUSTOM FIELDS")
print("="*60)

custom_fields = [
    ("legacy_service_id", "Legacy Service ID", "Int", 30),
    ("radius_user", "RADIUS User", "Data", 31),
    ("radius_password", "RADIUS Password", "Data", 32),
    ("product_sku", "Product SKU", "Data", 33),
    ("device", "Device", "Link", 34),  # options = Service Equipment
]

for fname, label, ftype, idx in custom_fields:
    exists = frappe.db.sql("""
        SELECT name FROM "tabDocField"
        WHERE parent = 'Service Subscription' AND fieldname = %s
    """, (fname,))
    if not exists:
        opts = "Service Equipment" if fname == "device" else None
        frappe.db.sql("""
            INSERT INTO "tabDocField" (
                name, creation, modified, modified_by, owner, docstatus, idx,
                parent, parentfield, parenttype,
                fieldname, label, fieldtype, options, reqd, read_only, hidden
            ) VALUES (
                %(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, %(idx)s,
                'Service Subscription', 'fields', 'DocType',
                %(fname)s, %(label)s, %(ftype)s, %(opts)s, 0, 0, 0
            )
        """, {
            "name": "ss-{}-{}".format(fname, int(time.time())),
            "now": now_str, "idx": idx,
            "fname": fname, "label": label, "ftype": ftype, "opts": opts,
        })
        # Add column to table
        col_type = "bigint" if ftype == "Int" else "varchar(140)"
        try:
            frappe.db.sql('ALTER TABLE "tabService Subscription" ADD COLUMN {} {}'.format(fname, col_type))
        except Exception as e:
            if "already exists" not in str(e).lower():
                raise
        frappe.db.commit()
        print("  Added field: {}".format(fname))
    else:
        print("  Field exists: {}".format(fname))

# ═══════════════════════════════════════════════════════════════
# PHASE 2: Build lookup maps
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 2: BUILD LOOKUP MAPS")
print("="*60)

# delivery_id → Service Location name
loc_map = {}
rows = frappe.db.sql("""
    SELECT name, legacy_delivery_id FROM "tabService Location"
    WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id > 0
""", as_dict=True)
for r in rows:
    loc_map[r["legacy_delivery_id"]] = r["name"]
print("Location map: {} entries".format(len(loc_map)))

# account_id → Customer name
cust_map = {}
rows = frappe.db.sql("""
    SELECT name, legacy_account_id FROM "tabCustomer"
    WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0
""", as_dict=True)
for r in rows:
    cust_map[r["legacy_account_id"]] = r["name"]
print("Customer map: {} entries".format(len(cust_map)))

# device_id → Service Equipment name
dev_map = {}
rows = frappe.db.sql("""
    SELECT name, legacy_device_id FROM "tabService Equipment"
    WHERE legacy_device_id IS NOT NULL AND legacy_device_id > 0
""", as_dict=True)
for r in rows:
    dev_map[r["legacy_device_id"]] = r["name"]
print("Device map: {} entries".format(len(dev_map)))

# delivery_id → account_id
with conn.cursor() as cur:
    cur.execute("SELECT id, account_id FROM delivery")
    del_acct = {}
    for r in cur.fetchall():
        del_acct[r["id"]] = r["account_id"]
print("Delivery→account map: {} entries".format(len(del_acct)))

# ═══════════════════════════════════════════════════════════════
# PHASE 3: Import services as Service Subscriptions
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 3: IMPORT SERVICE SUBSCRIPTIONS")
print("="*60)

# Clear existing
existing_subs = frappe.db.sql('SELECT COUNT(*) FROM "tabService Subscription"')[0][0]
if existing_subs > 0:
    frappe.db.sql('DELETE FROM "tabService Subscription"')
    frappe.db.commit()
    print("Deleted {} existing subscriptions".format(existing_subs))

# Product category → service_category mapping
# Legacy: Mensualités fibre, Installation fibre, Mensualités sans fil, Téléphonie,
#          Mensualités télévision, Installation télé, Adresse IP Fixe, Hébergement, etc.
PROD_CAT_MAP = {
    4: "Internet",    # Mensualités sans fil
    32: "Internet",   # Mensualités fibre
    8: "Internet",    # Installation et équipement internet sans fil
    26: "Internet",   # Installation et équipement fibre
    29: "Internet",   # Equipement internet fibre
    7: "Internet",    # Equipement internet sans fil
    23: "Internet",   # Internet camping
    17: "Internet",   # Adresse IP Fixe
    16: "Internet",   # Téléchargement supplémentaire
    21: "Internet",   # Location point à point
    33: "IPTV",       # Mensualités télévision
    34: "IPTV",       # Installation et équipement télé
    9: "VoIP",        # Téléphonie
    15: "Hébergement",  # Hébergement
    11: "Hébergement",  # Nom de domaine
    30: "Hébergement",  # Location espace cloud
    10: "Autre",      # Site internet
    13: "Autre",      # Location d'espace
}

# payment_recurrence → billing_cycle
RECUR_MAP = {
    1: "Mensuel",
    2: "Mensuel",
    3: "Trimestriel",
    4: "Annuel",
}

with conn.cursor() as cur:
    cur.execute("""
        SELECT s.id, s.delivery_id, s.device_id, s.product_id, s.status, s.comment,
               s.payment_recurrence, s.hijack, s.hijack_price,
               s.hijack_download_speed, s.hijack_upload_speed,
               s.date_orig, s.date_suspended, s.date_next_invoice, s.date_end_contract,
               s.forfait_internet, s.radius_user, s.radius_pwd,
               p.sku, p.price, p.download_speed, p.upload_speed, p.category as prod_cat
        FROM service s
        LEFT JOIN product p ON s.product_id = p.id
        WHERE s.status = 1
        ORDER BY s.id
    """)
    services = cur.fetchall()

    # Also get product names
    cur.execute("""
        SELECT pt.product_id, pt.name as prod_name
        FROM product_translate pt
        WHERE pt.language_id = 'francais'
    """)
    prod_names = {}
    for r in cur.fetchall():
        prod_names[r["product_id"]] = r["prod_name"]

print("Active services to import: {}".format(len(services)))

inserted = 0
no_location = 0

for svc in services:
    # Resolve customer via delivery → account
    customer = None
    service_location = loc_map.get(svc["delivery_id"]) if svc["delivery_id"] else None
    if not service_location:
        no_location += 1
        continue  # Skip services without a service location (required field)

    if svc["delivery_id"] and svc["delivery_id"] in del_acct:
        customer = cust_map.get(del_acct[svc["delivery_id"]])
    if not customer:
        no_location += 1
        continue  # Skip services without a customer (required field)

    # Service category
    prod_cat = svc["prod_cat"] or 0
    service_category = PROD_CAT_MAP.get(prod_cat, "Autre")

    # Plan name
    plan_name = html.unescape(prod_names.get(svc["product_id"], svc["sku"] or "Unknown"))

    # Speed (legacy stores in kbps, convert to Mbps)
    speed_down = 0
    speed_up = 0
    if svc["hijack"] and svc["hijack_download_speed"]:
        speed_down = int(svc["hijack_download_speed"]) // 1024
        speed_up = int(svc["hijack_upload_speed"] or 0) // 1024
    elif svc["download_speed"]:
        speed_down = int(svc["download_speed"]) // 1024
        speed_up = int(svc["upload_speed"] or 0) // 1024

    # Price
    price = float(svc["hijack_price"] or 0) if svc["hijack"] else float(svc["price"] or 0)

    # Billing cycle
    billing_cycle = RECUR_MAP.get(svc["payment_recurrence"], "Mensuel")

    # Start date
    start_date = None
    if svc["date_orig"]:
        try:
            start_date = datetime.fromtimestamp(int(svc["date_orig"])).strftime("%Y-%m-%d")
        except (ValueError, OSError):
            pass
    if not start_date:
        start_date = "2020-01-01"

    # End date (contract)
    end_date = None
    if svc["date_end_contract"]:
        try:
            end_date = datetime.fromtimestamp(int(svc["date_end_contract"])).strftime("%Y-%m-%d")
        except (ValueError, OSError):
            pass

    # Device link
    device = dev_map.get(svc["device_id"]) if svc["device_id"] else None

    # Generate name
    sub_name = "SUB-{}".format(svc["id"])

    inserted += 1
    frappe.db.sql("""
        INSERT INTO "tabService Subscription" (
            name, creation, modified, modified_by, owner, docstatus, idx,
            customer, service_location, status, service_category,
            plan_name, speed_down, speed_up,
            monthly_price, billing_cycle,
            start_date, end_date, notes,
            legacy_service_id, radius_user, radius_password, product_sku, device
        ) VALUES (
            %(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, 0,
            %(customer)s, %(service_location)s, 'Actif', %(service_category)s,
            %(plan_name)s, %(speed_down)s, %(speed_up)s,
            %(monthly_price)s, %(billing_cycle)s,
            %(start_date)s, %(end_date)s, %(notes)s,
            %(legacy_service_id)s, %(radius_user)s, %(radius_password)s, %(product_sku)s, %(device)s
        )
    """, {
        "name": sub_name,
        "now": now_str,
        "customer": customer,
        "service_location": service_location,
        "service_category": service_category,
        "plan_name": plan_name,
        "speed_down": speed_down,
        "speed_up": speed_up,
        "monthly_price": price,
        "billing_cycle": billing_cycle,
        "start_date": start_date,
        "end_date": end_date,
        "notes": svc["comment"] if svc["comment"] else None,
        "legacy_service_id": svc["id"],
        "radius_user": svc["radius_user"],
        "radius_password": svc["radius_pwd"],
        "product_sku": svc["sku"],
        "device": device,
    })

    if inserted % 5000 == 0:
        frappe.db.commit()
        print("  Inserted {}...".format(inserted))

frappe.db.commit()
print("Inserted {} Service Subscriptions ({} skipped - no location/customer)".format(inserted, no_location))

# ═══════════════════════════════════════════════════════════════
# PHASE 4: Enrich Customer records with account details
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 4: ENRICH CUSTOMER RECORDS")
print("="*60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, customer_id, email, email_autre, tel_home, cell,
               stripe_id, ppa, ppa_name, ppa_code, ppa_branch, ppa_account,
               notes_client, language_id, commercial, vip, mauvais_payeur,
               invoice_delivery, company, contact
        FROM account
        WHERE status = 1
    """)
    accounts = cur.fetchall()

print("Active accounts to enrich: {}".format(len(accounts)))

updated = 0
for acct in accounts:
    cust_name = cust_map.get(acct["id"])
    if not cust_name:
        continue

    # Build update fields
    updates = {}
    sets = []

    # Email
    if acct["email"] and acct["email"].strip():
        updates["email_id"] = acct["email"].strip()
        sets.append('email_id = %(email_id)s')

    # Mobile
    cell = (acct["cell"] or "").strip()
    if not cell:
        cell = (acct["tel_home"] or "").strip()
    if cell:
        updates["mobile_no"] = cell
        sets.append('mobile_no = %(mobile_no)s')

    # Stripe ID
    if acct["stripe_id"] and acct["stripe_id"].strip():
        updates["stripe_id"] = acct["stripe_id"].strip()
        sets.append('stripe_id = %(stripe_id)s')

    # PPA enabled
    if acct["ppa"]:
        updates["ppa_enabled"] = 1
        sets.append('ppa_enabled = %(ppa_enabled)s')

    # Language
    lang = "fr" if acct["language_id"] == "francais" else "en"
    updates["language"] = lang
    sets.append('language = %(language)s')

    # Customer details (notes + contact)
    details_parts = []
    if acct["notes_client"] and acct["notes_client"].strip():
        details_parts.append(acct["notes_client"].strip())
    if acct["contact"] and acct["contact"].strip():
        details_parts.append("Contact: " + acct["contact"].strip())
    if acct["vip"]:
        details_parts.append("[VIP]")
    if acct["mauvais_payeur"]:
        details_parts.append("[MAUVAIS PAYEUR]")
    if acct["commercial"]:
        details_parts.append("[COMMERCIAL]")
    if details_parts:
        updates["customer_details"] = "\n".join(details_parts)
        sets.append('customer_details = %(customer_details)s')

    if sets:
        updates["cust_name"] = cust_name
        frappe.db.sql(
            'UPDATE "tabCustomer" SET {} WHERE name = %(cust_name)s'.format(", ".join(sets)),
            updates
        )
        updated += 1

    if updated % 2000 == 0 and updated > 0:
        frappe.db.commit()
        print("  Updated {}...".format(updated))

frappe.db.commit()
print("Updated {} Customer records".format(updated))

conn.close()

# ═══════════════════════════════════════════════════════════════
# PHASE 5: VERIFY
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 5: VERIFY")
print("="*60)

total_subs = frappe.db.sql('SELECT COUNT(*) FROM "tabService Subscription"')[0][0]
by_cat = frappe.db.sql("""
    SELECT service_category, COUNT(*) as cnt FROM "tabService Subscription"
    GROUP BY service_category ORDER BY cnt DESC
""", as_dict=True)
print("Total Service Subscriptions: {}".format(total_subs))
for c in by_cat:
    print("  {}: {}".format(c["service_category"], c["cnt"]))

with_device = frappe.db.sql('SELECT COUNT(*) FROM "tabService Subscription" WHERE device IS NOT NULL')[0][0]
with_radius = frappe.db.sql('SELECT COUNT(*) FROM "tabService Subscription" WHERE radius_user IS NOT NULL')[0][0]
print("\nWith device link: {}".format(with_device))
print("With RADIUS credentials: {}".format(with_radius))

# Customer enrichment
cust_with_email = frappe.db.sql("SELECT COUNT(*) FROM \"tabCustomer\" WHERE email_id IS NOT NULL")[0][0]
cust_with_phone = frappe.db.sql("SELECT COUNT(*) FROM \"tabCustomer\" WHERE mobile_no IS NOT NULL")[0][0]
cust_with_stripe = frappe.db.sql("SELECT COUNT(*) FROM \"tabCustomer\" WHERE stripe_id IS NOT NULL")[0][0]
cust_with_ppa = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer" WHERE ppa_enabled = 1')[0][0]
print("\nCustomer enrichment:")
print("  With email: {}".format(cust_with_email))
print("  With phone: {}".format(cust_with_phone))
print("  With Stripe: {}".format(cust_with_stripe))
print("  With PPA: {}".format(cust_with_ppa))

# Sample subscriptions
samples = frappe.db.sql("""
    SELECT name, customer, service_category, plan_name, speed_down, speed_up,
           monthly_price, radius_user, device, legacy_service_id
    FROM "tabService Subscription" LIMIT 10
""", as_dict=True)
print("\nSample subscriptions:")
for s in samples:
    print("  {} cat={} plan={} {}↓/{}↑ ${} radius={} dev={}".format(
        s["name"], s["service_category"], (s["plan_name"] or "")[:30],
        s["speed_down"], s["speed_up"], s["monthly_price"],
        s["radius_user"] or "-", s["device"] or "-"))

elapsed = time.time() - T_TOTAL
print("\n" + "="*60)
print("DONE in {:.1f}s".format(elapsed))
print("="*60)

frappe.clear_cache()