gigafibre-fsm/scripts/migration/import_devices_and_enrich.py

"""
Import missing devices into Service Equipment and enrich Service Locations
with fibre data (connection_type, OLT port, VLANs).

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_devices_and_enrich.py
"""
import frappe
import pymysql
import os
import time
import hashlib
from datetime import datetime
from decimal import Decimal

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

T_TOTAL = time.time()
now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")

# ═══════════════════════════════════════════════════════════════
# LEGACY DB CONNECTION
# ═══════════════════════════════════════════════════════════════
conn = pymysql.connect(
    host="10.100.80.100",
    user="facturation",
    password="*******",
    database="gestionclient",
    cursorclass=pymysql.cursors.DictCursor
)

# ═══════════════════════════════════════════════════════════════
# PHASE 1: Build lookup maps
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 1: BUILD LOOKUP MAPS")
print("="*60)

# Map legacy delivery_id → ERPNext Service Location name
loc_map = {}
rows = frappe.db.sql("""
    SELECT name, legacy_delivery_id FROM "tabService Location"
    WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id > 0
""", as_dict=True)
for r in rows:
    loc_map[r["legacy_delivery_id"]] = r["name"]
print("Location map: {} entries".format(len(loc_map)))

# Map legacy account_id → ERPNext Customer name
cust_map = {}
rows = frappe.db.sql("""
    SELECT name, legacy_account_id FROM "tabCustomer"
    WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0
""", as_dict=True)
for r in rows:
    cust_map[r["legacy_account_id"]] = r["name"]
print("Customer map: {} entries".format(len(cust_map)))

# Get already-imported device IDs
existing_devices = set()
rows = frappe.db.sql("""
    SELECT legacy_device_id FROM "tabService Equipment"
    WHERE legacy_device_id IS NOT NULL AND legacy_device_id > 0
""")
for r in rows:
    existing_devices.add(r[0])
print("Already imported devices: {}".format(len(existing_devices)))

# Map delivery_id → account_id from legacy
with conn.cursor() as cur:
    cur.execute("SELECT id, account_id FROM delivery")
    delivery_account = {}
    for r in cur.fetchall():
        delivery_account[r["id"]] = r["account_id"]
print("Delivery→account map: {} entries".format(len(delivery_account)))

# ═══════════════════════════════════════════════════════════════
# PHASE 2: Import missing devices
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 2: IMPORT MISSING DEVICES")
print("="*60)

# Category mapping: legacy category → ERPNext equipment_type
CATEGORY_MAP = {
    "onu":              "ONT",
    "tplink_tplg":      "ONT",
    "tplink_device2":   "ONT",
    "raisecom_rcmg":    "ONT",
    "stb":              "Decodeur TV",
    "stb_ministra":     "Decodeur TV",
    "airosm":           "AP WiFi",
    "airos_ac":         "AP WiFi",
    "cambium":          "AP WiFi",
    "ht803g1ge":        "Telephone IP",
    "custom":           "Autre",
}

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, delivery_id, category, name, manufacturier, model,
               sn, mac, manage, port, protocol, manage_cli, port_cli,
               protocol_cli, user, pass, parent
        FROM device ORDER BY id
    """)
    devices = cur.fetchall()

print("Total legacy devices: {}".format(len(devices)))

# Build set of existing serial numbers to avoid unique constraint violations
existing_serials = frappe.db.sql("""
    SELECT serial_number FROM "tabService Equipment" WHERE serial_number IS NOT NULL
""")
seen_serials = set(r[0] for r in existing_serials)
print("Existing serial numbers: {}".format(len(seen_serials)))

inserted = 0
skipped = 0
batch = []

for dev in devices:
    if dev["id"] in existing_devices:
        skipped += 1
        continue

    equipment_type = CATEGORY_MAP.get(dev["category"], "Autre")
    serial_number = dev["sn"] or "NO-SN-{}".format(dev["id"])
    # Ensure uniqueness — append device ID if serial already seen
    if serial_number in seen_serials:
        serial_number = "{}-D{}".format(serial_number, dev["id"])
    seen_serials.add(serial_number)
    mac = dev["mac"] or None
    brand = dev["manufacturier"] or None
    model = dev["model"] or None

    # Management IP — prefer manage_cli (clean IP), fallback to manage (may be URL)
    ip_address = None
    if dev["manage_cli"] and dev["manage_cli"].strip():
        ip_address = dev["manage_cli"].strip()
    elif dev["manage"] and dev["manage"].strip():
        mgmt = dev["manage"].strip()
        # If it's just an IP (not a full URL), use it
        if not mgmt.startswith("http") and not "/" in mgmt:
            ip_address = mgmt

    login_user = dev["user"] if dev["user"] and dev["user"].strip() else None
    login_pass = dev["pass"] if dev["pass"] and dev["pass"].strip() else None

    # Link to Service Location via delivery_id
    service_location = loc_map.get(dev["delivery_id"]) if dev["delivery_id"] else None

    # Link to Customer via delivery → account
    customer = None
    if dev["delivery_id"] and dev["delivery_id"] in delivery_account:
        account_id = delivery_account[dev["delivery_id"]]
        customer = cust_map.get(account_id)

    # Generate unique name
    eq_name = "EQ-{}".format(hashlib.md5(str(dev["id"]).encode()).hexdigest()[:10])

    batch.append({
        "name": eq_name,
        "now": now_str,
        "equipment_type": equipment_type,
        "serial_number": serial_number,
        "mac_address": mac,
        "brand": brand,
        "model": model,
        "ip_address": ip_address,
        "login_user": login_user,
        "login_password": login_pass,
        "customer": customer,
        "service_location": service_location,
        "legacy_device_id": dev["id"],
        "status": "Actif",
        "ownership": "Gigafibre",
    })
    inserted += 1

    # Insert in batches of 500
    if len(batch) >= 500:
        for eq in batch:
            frappe.db.sql("""
                INSERT INTO "tabService Equipment" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    equipment_type, serial_number, mac_address, brand, model,
                    ip_address, login_user, login_password,
                    customer, service_location, legacy_device_id,
                    status, ownership
                ) VALUES (
                    %(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, 0,
                    %(equipment_type)s, %(serial_number)s, %(mac_address)s, %(brand)s, %(model)s,
                    %(ip_address)s, %(login_user)s, %(login_password)s,
                    %(customer)s, %(service_location)s, %(legacy_device_id)s,
                    %(status)s, %(ownership)s
                )
            """, eq)
        frappe.db.commit()
        batch = []
        print("  Inserted {}...".format(inserted))

# Final batch
if batch:
    for eq in batch:
        frappe.db.sql("""
            INSERT INTO "tabService Equipment" (
                name, creation, modified, modified_by, owner, docstatus, idx,
                equipment_type, serial_number, mac_address, brand, model,
                ip_address, login_user, login_password,
                customer, service_location, legacy_device_id,
                status, ownership
            ) VALUES (
                %(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, 0,
                %(equipment_type)s, %(serial_number)s, %(mac_address)s, %(brand)s, %(model)s,
                %(ip_address)s, %(login_user)s, %(login_password)s,
                %(customer)s, %(service_location)s, %(legacy_device_id)s,
                %(status)s, %(ownership)s
            )
        """, eq)
    frappe.db.commit()

print("Inserted {} new devices ({} already existed)".format(inserted, skipped))

# ═══════════════════════════════════════════════════════════════
# PHASE 3: Enrich Service Locations with fibre data
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 3: ENRICH LOCATIONS WITH FIBRE DATA")
print("="*60)

with conn.cursor() as cur:
    # Get fibre data with service → delivery link
    cur.execute("""
        SELECT f.service_id, s.delivery_id,
               f.frame, f.slot, f.port, f.ontid,
               f.vlan_manage, f.vlan_internet, f.vlan_telephone, f.vlan_tele,
               f.sn as ont_sn, f.tech as fibre_tech
        FROM fibre f
        LEFT JOIN service s ON f.service_id = s.id
        WHERE s.delivery_id IS NOT NULL
    """)
    fibre_data = cur.fetchall()

print("Fibre records with delivery link: {}".format(len(fibre_data)))

updated_locs = 0
for fb in fibre_data:
    loc_name = loc_map.get(fb["delivery_id"])
    if not loc_name:
        continue

    olt_port = "{}/{}/{}".format(fb["frame"], fb["slot"], fb["port"])
    if fb["ontid"]:
        olt_port += " ONT:{}".format(fb["ontid"])

    vlans = []
    if fb["vlan_internet"]:
        vlans.append("inet:{}".format(fb["vlan_internet"]))
    if fb["vlan_manage"]:
        vlans.append("mgmt:{}".format(fb["vlan_manage"]))
    if fb["vlan_telephone"]:
        vlans.append("tel:{}".format(fb["vlan_telephone"]))
    if fb["vlan_tele"]:
        vlans.append("tv:{}".format(fb["vlan_tele"]))
    network_id = " ".join(vlans) if vlans else None

    frappe.db.sql("""
        UPDATE "tabService Location"
        SET connection_type = 'Fibre FTTH',
            olt_port = %(olt_port)s,
            network_id = %(network_id)s
        WHERE name = %(name)s
          AND (connection_type IS NULL OR connection_type = '')
    """, {"name": loc_name, "olt_port": olt_port, "network_id": network_id})
    updated_locs += 1

frappe.db.commit()
print("Enriched {} locations with fibre data".format(updated_locs))

# Also set connection_type for locations with wireless devices
with conn.cursor() as cur:
    cur.execute("""
        SELECT DISTINCT d.delivery_id
        FROM device d
        WHERE d.category IN ('airosm', 'airos_ac', 'cambium')
          AND d.delivery_id > 0
    """)
    wireless_deliveries = [r["delivery_id"] for r in cur.fetchall()]

wireless_updated = 0
for del_id in wireless_deliveries:
    loc_name = loc_map.get(del_id)
    if loc_name:
        frappe.db.sql("""
            UPDATE "tabService Location"
            SET connection_type = 'Sans-fil'
            WHERE name = %s
              AND (connection_type IS NULL OR connection_type = '')
        """, (loc_name,))
        wireless_updated += 1

frappe.db.commit()
print("Set {} locations as Sans-fil (wireless)".format(wireless_updated))

conn.close()

# ═══════════════════════════════════════════════════════════════
# PHASE 4: VERIFY
# ═══════════════════════════════════════════════════════════════
print("\n" + "="*60)
print("PHASE 4: VERIFY")
print("="*60)

total_eq = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment"')[0][0]
by_type = frappe.db.sql("""
    SELECT equipment_type, COUNT(*) as cnt FROM "tabService Equipment"
    GROUP BY equipment_type ORDER BY cnt DESC
""", as_dict=True)
print("Total Service Equipment: {}".format(total_eq))
for t in by_type:
    print("  {}: {}".format(t["equipment_type"], t["cnt"]))

with_customer = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment" WHERE customer IS NOT NULL')[0][0]
with_location = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment" WHERE service_location IS NOT NULL')[0][0]
with_ip = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Equipment\" WHERE ip_address IS NOT NULL")[0][0]
with_creds = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Equipment\" WHERE login_user IS NOT NULL")[0][0]
print("\nWith customer link: {}".format(with_customer))
print("With service_location link: {}".format(with_location))
print("With IP address: {}".format(with_ip))
print("With credentials: {}".format(with_creds))

# Location enrichment stats
loc_by_conn = frappe.db.sql("""
    SELECT connection_type, COUNT(*) as cnt FROM "tabService Location"
    GROUP BY connection_type ORDER BY cnt DESC
""", as_dict=True)
print("\nService Locations by connection type:")
for l in loc_by_conn:
    print("  {}: {}".format(l["connection_type"] or "(not set)", l["cnt"]))

with_olt = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Location\" WHERE olt_port IS NOT NULL")[0][0]
print("With OLT port: {}".format(with_olt))

elapsed = time.time() - T_TOTAL
print("\n" + "="*60)
print("DONE in {:.1f}s".format(elapsed))
print("="*60)