gigafibre-fsm/scripts/migration/import_missing_services.py

"""
Import missing services — categories excluded from the original migration.

Original migration (phase 3) only imported categories: 4,9,17,21,32,33
This imports ALL remaining active services from other categories.

For each missing service:
  1. Ensure Item exists in ERPNext
  2. Ensure Subscription Plan exists
  3. Create Subscription linked to correct customer + service_location

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_missing_services.py
"""
import frappe
import pymysql
import os
from datetime import datetime, timezone

DRY_RUN = False

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
print("Connected:", frappe.local.site)

conn = pymysql.connect(
    host="legacy-db", user="facturation", password="VD67owoj",
    database="gestionclient", cursorclass=pymysql.cursors.DictCursor
)

# ═══════════════════════════════════════════════════════════════
# LOAD LEGACY DATA
# ═══════════════════════════════════════════════════════════════
print("Loading legacy data...")
with conn.cursor() as cur:
    # All active services from excluded categories
    cur.execute("""
        SELECT s.id, s.delivery_id, s.product_id, s.status,
               s.hijack, s.hijack_price, s.hijack_desc,
               s.payment_recurrence, s.date_orig, s.date_next_invoice,
               s.radius_user, s.radius_pwd,
               p.sku, p.price as base_price, p.category,
               d.account_id
        FROM service s
        JOIN product p ON p.id = s.product_id
        JOIN delivery d ON d.id = s.delivery_id
        WHERE s.status = 1
        AND p.category NOT IN (4,9,17,21,32,33)
        ORDER BY d.account_id, s.id
    """)
    missing_services = cur.fetchall()

    # All products from these categories
    cur.execute("""
        SELECT DISTINCT p.id, p.sku, p.price, p.category
        FROM product p
        JOIN service s ON s.product_id = p.id
        WHERE s.status = 1 AND p.category NOT IN (4,9,17,21,32,33)
    """)
    products = cur.fetchall()

    # Category names (hardcoded from legacy)
    categories = {
        1: "Installation initiale", 7: "Location serveur", 8: "Location équipement",
        11: "Nom de domaine", 13: "Location espace", 15: "Hébergement",
        16: "Support", 23: "Hotspot camping", 26: "Installation et équipement fibre",
        28: "Quotidien pro", 34: "Installation et équipement télé",
    }

conn.close()
print("Missing active services: {}".format(len(missing_services)))
print("Distinct products: {}".format(len(products)))

# Show breakdown by category
cat_counts = {}
for s in missing_services:
    cat = s["category"]
    cat_name = categories.get(cat, "cat={}".format(cat))
    if cat_name not in cat_counts:
        cat_counts[cat_name] = 0
    cat_counts[cat_name] += 1
for name, cnt in sorted(cat_counts.items(), key=lambda x: -x[1]):
    print("  {}: {}".format(name, cnt))

# ═══════════════════════════════════════════════════════════════
# LOAD ERPNEXT DATA
# ═══════════════════════════════════════════════════════════════
print("\nLoading ERPNext data...")

# Existing items
existing_items = set()
items = frappe.db.sql('SELECT name FROM "tabItem"', as_dict=True)
for i in items:
    existing_items.add(i["name"])
print("Existing items: {}".format(len(existing_items)))

# Existing subscription plans
existing_plans = set()
plans = frappe.db.sql('SELECT plan_name FROM "tabSubscription Plan"', as_dict=True)
for p in plans:
    existing_plans.add(p["plan_name"])
print("Existing plans: {}".format(len(existing_plans)))

# Item details: sku → {item_name, item_group}
item_details = {}
item_rows = frappe.db.sql('SELECT name, item_name, item_group FROM "tabItem"', as_dict=True)
for i in item_rows:
    item_details[i["name"]] = {"item_name": i["item_name"], "item_group": i["item_group"]}

# Existing subscriptions by legacy_service_id
existing_subs = set()
subs = frappe.db.sql('SELECT legacy_service_id FROM "tabSubscription" WHERE legacy_service_id IS NOT NULL', as_dict=True)
for s in subs:
    existing_subs.add(s["legacy_service_id"])
print("Existing subscriptions: {}".format(len(existing_subs)))

# Customer mapping: legacy_account_id → ERPNext customer name
cust_map = {}
custs = frappe.db.sql('SELECT name, legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0', as_dict=True)
for c in custs:
    cust_map[c["legacy_account_id"]] = c["name"]
print("Customer mapping: {}".format(len(cust_map)))

# Service location mapping: look up by customer + delivery address
# We'll find the service_location from existing subscriptions for the same delivery
loc_map = {}  # (account_id, delivery_id) → service_location from existing subs
loc_subs = frappe.db.sql("""
    SELECT s.legacy_service_id, s.service_location, s.party
    FROM "tabSubscription" s
    WHERE s.service_location IS NOT NULL AND s.legacy_service_id IS NOT NULL
""", as_dict=True)
# Build delivery → location map from legacy
with pymysql.connect(host="legacy-db", user="facturation", password="VD67owoj",
    database="gestionclient", cursorclass=pymysql.cursors.DictCursor) as conn2:
    with conn2.cursor() as cur2:
        cur2.execute("""
            SELECT s.id as service_id, s.delivery_id, d.account_id
            FROM service s
            JOIN delivery d ON d.id = s.delivery_id
            WHERE s.status = 1
        """)
        svc_delivery = {r["service_id"]: (r["account_id"], r["delivery_id"]) for r in cur2.fetchall()}

# Map delivery_id → service_location from existing subscriptions
delivery_loc = {}
for ls in loc_subs:
    sid = ls["legacy_service_id"]
    if sid in svc_delivery:
        acct, did = svc_delivery[sid]
        if did not in delivery_loc and ls["service_location"]:
            delivery_loc[did] = ls["service_location"]
print("Delivery→location mappings: {}".format(len(delivery_loc)))

# Category to item_group mapping
CATEGORY_GROUP = {
    26: "Installation et équipement fibre",
    34: "Installation et équipement télé",
    8: "Installation et équipement fibre",
    15: "Hébergement",
    11: "Nom de domaine",
    13: "Installation et équipement fibre",
    1: "Installation et équipement fibre",
    16: "Installation et équipement fibre",
    28: "Installation et équipement fibre",
    7: "Installation et équipement fibre",
    23: "Installation et équipement fibre",
}

TAX_TEMPLATE = "Canada - Résidentiel - TC"
COMPANY = "Targo Communications"

def ts_to_date(ts):
    if not ts or ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
    except (ValueError, OSError):
        return None

# ═══════════════════════════════════════════════════════════════
# PREPARE: Items and Plans
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("ITEMS & PLANS")
print("=" * 70)

items_to_create = []
plans_to_create = []

for p in products:
    sku = p["sku"]
    if sku not in existing_items:
        items_to_create.append({
            "sku": sku,
            "price": float(p["price"]),
            "category": p["category"],
            "group": CATEGORY_GROUP.get(p["category"], "Installation et équipement fibre"),
        })

    plan_name = "PLAN-" + sku
    if plan_name not in existing_plans:
        plans_to_create.append({
            "plan_name": plan_name,
            "item": sku,
            "cost": float(p["price"]),
        })

print("Items to create: {}".format(len(items_to_create)))
for i in items_to_create:
    print("  {} @ {:.2f} → {}".format(i["sku"], i["price"], i["group"]))

print("Plans to create: {}".format(len(plans_to_create)))
for p in plans_to_create:
    print("  {} (item: {}, cost: {:.2f})".format(p["plan_name"], p["item"], p["cost"]))

# ═══════════════════════════════════════════════════════════════
# PREPARE: Subscriptions
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 70)
print("SUBSCRIPTIONS")
print("=" * 70)

subs_to_create = []
skipped = {"no_customer": 0, "already_exists": 0, "no_location": 0}

for s in missing_services:
    sid = s["id"]
    if sid in existing_subs:
        skipped["already_exists"] += 1
        continue

    acct_id = s["account_id"]
    cust_name = cust_map.get(acct_id)
    if not cust_name:
        skipped["no_customer"] += 1
        continue

    sku = s["sku"]
    plan_name = "PLAN-" + sku
    delivery_id = s["delivery_id"]
    service_location = delivery_loc.get(delivery_id)

    if not service_location:
        skipped["no_location"] += 1
        # Still create — just without location
        pass

    price = float(s["hijack_price"]) if s["hijack"] else float(s["base_price"])
    freq = "A" if s["payment_recurrence"] == 0 else "M"

    start_date = ts_to_date(s["date_orig"]) or "2020-01-01"

    idet = item_details.get(sku, {})
    subs_to_create.append({
        "legacy_id": sid,
        "customer": cust_name,
        "plan": plan_name,
        "sku": sku,
        "price": price,
        "freq": freq,
        "start_date": start_date,
        "service_location": service_location,
        "hijack_desc": s["hijack_desc"] or "",
        "category": s["category"],
        "item_name": idet.get("item_name", sku),
        "item_group": idet.get("item_group", ""),
    })

print("Subscriptions to create: {}".format(len(subs_to_create)))
print("Skipped: {}".format(skipped))

# Show samples by category
for cat_name in sorted(cat_counts.keys(), key=lambda x: -cat_counts[x]):
    samples = [s for s in subs_to_create if categories.get(s["category"]) == cat_name][:3]
    if samples:
        print("\n  {} ({} total):".format(cat_name, sum(1 for s in subs_to_create if categories.get(s["category"]) == cat_name)))
        for s in samples:
            print("    svc#{} {} {} {:.2f} → {} at {}".format(
                s["legacy_id"], s["sku"], s["freq"], s["price"],
                s["customer"], s["service_location"] or "NO_LOC"))

# ═══════════════════════════════════════════════════════════════
# APPLY
# ═══════════════════════════════════════════════════════════════
if DRY_RUN:
    print("\n*** DRY RUN — no changes made ***")
    print("Set DRY_RUN = False to create {} items, {} plans, {} subscriptions".format(
        len(items_to_create), len(plans_to_create), len(subs_to_create)))
else:
    print("\n" + "=" * 70)
    print("APPLYING CHANGES")
    print("=" * 70)

    # Step 1: Create missing Items
    for i in items_to_create:
        try:
            item_groups = frappe.db.sql('SELECT name FROM "tabItem Group" WHERE name = %s', (i["group"],))
            if not item_groups:
                i["group"] = "All Item Groups"

            doc = frappe.get_doc({
                "doctype": "Item",
                "item_code": i["sku"],
                "item_name": i["sku"],
                "item_group": i["group"],
                "stock_uom": "Nos",
                "is_stock_item": 0,
            })
            doc.insert(ignore_permissions=True)
            print("  Created item: {}".format(i["sku"]))
        except Exception as e:
            print("  ERR item {}: {}".format(i["sku"], str(e)[:100]))
    frappe.db.commit()

    # Step 2: Create missing Subscription Plans
    for p in plans_to_create:
        try:
            doc = frappe.get_doc({
                "doctype": "Subscription Plan",
                "plan_name": p["plan_name"],
                "item": p["item"],
                "price_determination": "Fixed Rate",
                "cost": p["cost"],
                "currency": "CAD",
                "billing_interval": "Month",
                "billing_interval_count": 1,
            })
            doc.insert(ignore_permissions=True)
            print("  Created plan: {}".format(p["plan_name"]))
        except Exception as e:
            print("  ERR plan {}: {}".format(p["plan_name"], str(e)[:100]))
    frappe.db.commit()

    # Step 3: Create subscriptions
    created = 0
    errors = 0
    for s in subs_to_create:
        try:
            sub = frappe.get_doc({
                "doctype": "Subscription",
                "party_type": "Customer",
                "party": s["customer"],
                "company": COMPANY,
                "status": "Active",
                "start_date": s["start_date"],
                "generate_invoice_at": "Beginning of the current subscription period",
                "days_until_due": 30,
                "follow_calendar_months": 1,
                "generate_new_invoices_past_due_date": 1,
                "submit_invoice": 0,
                "cancel_at_period_end": 0,
                "legacy_service_id": s["legacy_id"],
                "service_location": s["service_location"],
                "actual_price": s["price"],
                "custom_description": s["hijack_desc"] if s["hijack_desc"] else None,
                "item_code": s["sku"],
                "item_name": s["item_name"],
                "item_group": s["item_group"],
                "billing_frequency": s["freq"],
                "plans": [{
                    "plan": s["plan"],
                    "qty": 1,
                }],
            })
            sub.flags.ignore_validate = True
            sub.flags.ignore_links = True
            sub.insert(ignore_permissions=True)
            created += 1

            if created % 500 == 0:
                frappe.db.commit()
                print("  Progress: {}/{}".format(created, len(subs_to_create)))

        except Exception as e:
            errors += 1
            if errors <= 20:
                print("  ERR svc#{}: {}".format(s["legacy_id"], str(e)[:150]))

    frappe.db.commit()
    print("\nCreated: {} subscriptions".format(created))
    print("Errors: {}".format(errors))

print("\n" + "=" * 70)
print("DONE")
print("=" * 70)