gigafibre-fsm/scripts/migration/migrate_missing_data.py

"""
migrate_missing_data.py — Populate the new custom fields from legacy DB.

Prerequisites: add_missing_custom_fields.py must have been run first.

Migrates:
  1. Customer: PPA, Stripe, VIP, termination, portal, flags
  2. Service Subscription: hijack details, dates, quotas
  3. Service Equipment: management access, parent hierarchy, legacy category
  4. Service Location: fibre detail (VLANs individual, OLT IP/name, ONT ID, etc.)
  5. Dispatch Job: tech times from bon_travail

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/migrate_missing_data.py
"""
import frappe
import pymysql
import os
import json
import time
from datetime import datetime

os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

conn = pymysql.connect(
    host="legacy-db",
    user="facturation",
    password="VD67owoj",
    database="gestionclient",
    cursorclass=pymysql.cursors.DictCursor
)

def ts_to_date(ts):
    """Convert unix timestamp to YYYY-MM-DD, return None on failure."""
    if not ts or ts == 0:
        return None
    try:
        return datetime.fromtimestamp(int(ts)).strftime("%Y-%m-%d")
    except (ValueError, OSError, OverflowError):
        return None


# ═══════════════════════════════════════════════════════════════
# BUILD MAPS
# ═══════════════════════════════════════════════════════════════
print("Building lookup maps...")

# legacy_account_id → ERPNext Customer name
cust_map = {}
for r in frappe.db.sql('SELECT name, legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0', as_dict=True):
    cust_map[r["legacy_account_id"]] = r["name"]
print("  Customer map: {}".format(len(cust_map)))

# legacy_delivery_id → ERPNext Service Location name
loc_map = {}
for r in frappe.db.sql('SELECT name, legacy_delivery_id FROM "tabService Location" WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id > 0', as_dict=True):
    loc_map[r["legacy_delivery_id"]] = r["name"]
print("  Location map: {}".format(len(loc_map)))

# legacy_service_id → ERPNext Service Subscription name
sub_map = {}
for r in frappe.db.sql('SELECT name, legacy_service_id FROM "tabService Subscription" WHERE legacy_service_id IS NOT NULL AND legacy_service_id > 0', as_dict=True):
    sub_map[r["legacy_service_id"]] = r["name"]
print("  Subscription map: {}".format(len(sub_map)))

# legacy_device_id → ERPNext Service Equipment name
dev_map = {}
for r in frappe.db.sql('SELECT name, legacy_device_id FROM "tabService Equipment" WHERE legacy_device_id IS NOT NULL AND legacy_device_id > 0', as_dict=True):
    dev_map[r["legacy_device_id"]] = r["name"]
print("  Device map: {}".format(len(dev_map)))


# ═══════════════════════════════════════════════════════════════
# 1. CUSTOMER — PPA, Stripe, VIP, termination, portal, flags
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("1. CUSTOMER — PPA, Stripe, VIP, termination, portal")
print("=" * 60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, middle_name, title,
               ppa, ppa_name, ppa_code, ppa_branch, ppa_account,
               ppa_amount, ppa_amount_buffer, ppa_fixed, ppa_cc, ppa_all_invoice,
               stripe_id, stripe_ppa,
               vip, land_owner, pub, `call`,
               terminate_reason, terminate_cie, terminate_note, terminate_date,
               notes_client,
               username, password, keyword
        FROM account
    """)
    accounts = cur.fetchall()

updated = 0
for acct in accounts:
    cust_name = cust_map.get(acct["id"])
    if not cust_name:
        continue

    sets = []
    params = {"name": cust_name}

    # PPA fields
    if acct["ppa"]:
        sets.append("ppa_enabled = 1")
    for f in ["ppa_name", "ppa_code", "ppa_branch", "ppa_account"]:
        if acct.get(f):
            sets.append("{} = %({})s".format(f, f))
            params[f] = str(acct[f]).strip() if acct[f] else None
    if acct["ppa_amount"]:
        sets.append("ppa_amount = %(ppa_amount)s")
        params["ppa_amount"] = float(acct["ppa_amount"])
    if acct["ppa_amount_buffer"]:
        sets.append("ppa_amount_buffer = %(ppa_amount_buffer)s")
        params["ppa_amount_buffer"] = float(acct["ppa_amount_buffer"])
    if acct["ppa_fixed"]:
        sets.append("ppa_fixed = 1")
    if acct["ppa_cc"]:
        sets.append("ppa_cc = 1")
    if acct["ppa_all_invoice"]:
        sets.append("ppa_all_invoice = 1")

    # Stripe
    if acct["stripe_id"]:
        sets.append("stripe_customer_id = %(stripe_id)s")
        params["stripe_id"] = str(acct["stripe_id"]).strip()
    if acct["stripe_ppa"]:
        sets.append("stripe_ppa_enabled = 1")

    # Flags
    if acct["vip"]:
        sets.append("is_vip = 1")
    if acct["land_owner"]:
        sets.append("is_land_owner = 1")
    if acct["pub"]:
        sets.append("marketing_optin = 1")
    if acct.get("call"):
        sets.append("call_contact = 1")

    # Termination
    if acct["terminate_reason"]:
        sets.append("terminate_reason = %(terminate_reason)s")
        params["terminate_reason"] = str(acct["terminate_reason"])
    if acct["terminate_cie"]:
        sets.append("terminate_cie = %(terminate_cie)s")
        params["terminate_cie"] = str(acct["terminate_cie"])
    if acct["terminate_note"]:
        sets.append("terminate_note = %(terminate_note)s")
        params["terminate_note"] = str(acct["terminate_note"])
    if acct["terminate_date"]:
        td = ts_to_date(acct["terminate_date"])
        if td:
            sets.append("terminate_date = %(terminate_date)s")
            params["terminate_date"] = td

    # Client notes
    if acct["notes_client"]:
        sets.append("notes_client = %(notes_client)s")
        params["notes_client"] = str(acct["notes_client"])

    # Portal
    if acct["username"]:
        sets.append("portal_username = %(portal_username)s")
        params["portal_username"] = str(acct["username"])
    if acct["password"]:
        sets.append("portal_password_hash = %(portal_password_hash)s")
        params["portal_password_hash"] = str(acct["password"])

    # Misc
    if acct["middle_name"]:
        sets.append("middle_name = %(middle_name)s")
        params["middle_name"] = str(acct["middle_name"]).strip()
    if acct["title"]:
        sets.append("salutation_legacy = %(salutation_legacy)s")
        params["salutation_legacy"] = str(acct["title"]).strip()
    if acct["keyword"]:
        sets.append("search_keyword = %(search_keyword)s")
        params["search_keyword"] = str(acct["keyword"]).strip()

    if sets:
        sql = 'UPDATE "tabCustomer" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated += 1

frappe.db.commit()
print("Updated {} customers".format(updated))


# ═══════════════════════════════════════════════════════════════
# 2. SERVICE SUBSCRIPTION — hijack details, dates, quotas
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("2. SERVICE SUBSCRIPTION — hijack, dates, IP")
print("=" * 60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, hijack, hijack_desc,
               hijack_quota_day, hijack_quota_night,
               date_suspended, actif_until, date_next_invoice,
               forfait_internet, radius_conso, ip_fixe
        FROM service
    """)
    services = cur.fetchall()

updated = 0
for svc in services:
    sub_name = sub_map.get(svc["id"])
    if not sub_name:
        continue

    sets = []
    params = {"name": sub_name}

    if svc["hijack"]:
        sets.append("is_custom_pricing = 1")
    if svc["hijack_desc"]:
        sets.append("custom_pricing_desc = %(hijack_desc)s")
        params["hijack_desc"] = str(svc["hijack_desc"])
    if svc["hijack_quota_day"]:
        sets.append("quota_day_gb = %(quota_day)s")
        params["quota_day"] = float(svc["hijack_quota_day"])
    if svc["hijack_quota_night"]:
        sets.append("quota_night_gb = %(quota_night)s")
        params["quota_night"] = float(svc["hijack_quota_night"])

    ds = ts_to_date(svc["date_suspended"])
    if ds:
        sets.append("date_suspended = %(date_suspended)s")
        params["date_suspended"] = ds
    au = ts_to_date(svc["actif_until"])
    if au:
        sets.append("active_until = %(active_until)s")
        params["active_until"] = au
    ni = ts_to_date(svc["date_next_invoice"])
    if ni:
        sets.append("next_invoice_date = %(next_invoice_date)s")
        params["next_invoice_date"] = ni

    if svc["forfait_internet"]:
        sets.append("forfait_internet = 1")
    if svc["radius_conso"]:
        sets.append("radius_consumption = %(radius_conso)s")
        params["radius_conso"] = str(svc["radius_conso"])
    if svc.get("ip_fixe"):
        sets.append("static_ip = %(static_ip)s")
        params["static_ip"] = str(svc["ip_fixe"])

    if sets:
        sql = 'UPDATE "tabService Subscription" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated += 1

frappe.db.commit()
print("Updated {} subscriptions".format(updated))


# ═══════════════════════════════════════════════════════════════
# 3. SERVICE EQUIPMENT — management, parent, category, device_attr
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("3. SERVICE EQUIPMENT — management, parent, category")
print("=" * 60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT id, manage, port, protocol, manage_cli, port_cli, protocol_cli,
               parent, category, name as device_name
        FROM device
    """)
    devices = cur.fetchall()

updated = 0
for dev in devices:
    eq_name = dev_map.get(dev["id"])
    if not eq_name:
        continue

    sets = []
    params = {"name": eq_name}

    if dev["manage"]:
        sets.append("manage_url = %(manage_url)s")
        params["manage_url"] = str(dev["manage"]).strip()
    if dev["port"]:
        sets.append("manage_port = %(manage_port)s")
        params["manage_port"] = int(dev["port"])
    if dev["protocol"]:
        proto = str(dev["protocol"]).strip().upper()
        if proto in ("HTTP", "HTTPS", "SSH", "TELNET", "SNMP"):
            sets.append("manage_protocol = %(manage_protocol)s")
            params["manage_protocol"] = proto
    if dev["manage_cli"]:
        sets.append("cli_ip = %(cli_ip)s")
        params["cli_ip"] = str(dev["manage_cli"]).strip()
    if dev["port_cli"]:
        sets.append("cli_port = %(cli_port)s")
        params["cli_port"] = int(dev["port_cli"])
    if dev["protocol_cli"]:
        proto_cli = str(dev["protocol_cli"]).strip().upper()
        if proto_cli in ("SSH", "TELNET"):
            sets.append("cli_protocol = %(cli_protocol)s")
            params["cli_protocol"] = proto_cli
    if dev["parent"]:
        parent_eq = dev_map.get(dev["parent"])
        if parent_eq:
            sets.append("parent_device = %(parent_device)s")
            params["parent_device"] = parent_eq
    if dev["category"]:
        sets.append("legacy_category = %(legacy_category)s")
        params["legacy_category"] = str(dev["category"])
    if dev["device_name"]:
        sets.append("device_name_legacy = %(device_name_legacy)s")
        params["device_name_legacy"] = str(dev["device_name"])

    if sets:
        sql = 'UPDATE "tabService Equipment" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated += 1

frappe.db.commit()
print("Updated {} devices".format(updated))

# ── device_attr → JSON MAC addresses + stb_id ──
print("\nImporting device_attr...")
with conn.cursor() as cur:
    cur.execute("SELECT device_id, `key`, value FROM device_attr ORDER BY device_id")
    attrs = cur.fetchall()

# Group by device_id
device_attrs = {}
for attr in attrs:
    did = attr["device_id"]
    if did not in device_attrs:
        device_attrs[did] = {}
    device_attrs[did][attr["key"]] = attr["value"]

updated_attrs = 0
for did, kv in device_attrs.items():
    eq_name = dev_map.get(did)
    if not eq_name:
        continue

    sets = []
    params = {"name": eq_name}

    # Extract MAC addresses
    macs = {}
    stb_id = None
    for k, v in kv.items():
        if k.startswith("mac_") or k.startswith("eth"):
            macs[k] = v
        if k == "stb_id":
            stb_id = v

    if macs:
        sets.append("mac_addresses_json = %(macs_json)s")
        params["macs_json"] = json.dumps(macs)
    if stb_id:
        sets.append("iptv_subscription_id = %(stb_id)s")
        params["stb_id"] = str(stb_id)

    if sets:
        sql = 'UPDATE "tabService Equipment" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated_attrs += 1

frappe.db.commit()
print("Updated {} devices with attrs".format(updated_attrs))


# ═══════════════════════════════════════════════════════════════
# 4. SERVICE LOCATION — fibre detail (VLANs, OLT IP, ONT, etc.)
# ═══════════════════════════════════════════════════════════════
print("\n" + "=" * 60)
print("4. SERVICE LOCATION — fibre detail, VLANs, building")
print("=" * 60)

with conn.cursor() as cur:
    cur.execute("""
        SELECT f.id as fibre_id, f.service_id, f.sn as ont_sn,
               f.info_connect as olt_ip, f.ontid, f.tech as terrain,
               f.distance, f.nb_portees, f.temps_estim, f.suite,
               f.boitier_pas_install,
               f.vlan_manage, f.vlan_internet, f.vlan_telephone, f.vlan_tele,
               f.manage_service_id, f.internet_service_id,
               f.telephone_service_id, f.tele_service_id,
               f.placemarks_id, f.appartements_id,
               fo.description as olt_name,
               s.delivery_id
        FROM fibre f
        LEFT JOIN fibre_olt fo ON f.info_connect = fo.ip
        LEFT JOIN service s ON f.service_id = s.id
        WHERE s.delivery_id IS NOT NULL
    """)
    fibres = cur.fetchall()

updated = 0
for fb in fibres:
    loc_name = loc_map.get(fb["delivery_id"])
    if not loc_name:
        continue

    sets = []
    params = {"name": loc_name}

    if fb["fibre_id"]:
        sets.append("legacy_fibre_id = %(fibre_id)s")
        params["fibre_id"] = fb["fibre_id"]
    if fb["ont_sn"]:
        sets.append("ont_serial = %(ont_sn)s")
        params["ont_sn"] = str(fb["ont_sn"])
    if fb["olt_ip"]:
        sets.append("olt_ip = %(olt_ip)s")
        params["olt_ip"] = str(fb["olt_ip"])
    if fb["olt_name"]:
        sets.append("olt_name = %(olt_name)s")
        params["olt_name"] = str(fb["olt_name"])
    if fb["ontid"]:
        sets.append("ont_id = %(ontid)s")
        params["ontid"] = int(fb["ontid"])
    if fb["terrain"]:
        sets.append("terrain_type = %(terrain)s")
        params["terrain"] = str(fb["terrain"])
    if fb["distance"]:
        sets.append("fibre_distance_m = %(distance)s")
        params["distance"] = float(fb["distance"])
    if fb["nb_portees"]:
        sets.append("fibre_spans = %(spans)s")
        params["spans"] = int(fb["nb_portees"])
    if fb["temps_estim"]:
        sets.append("install_time_estimate = %(temps)s")
        params["temps"] = str(fb["temps_estim"])
    if fb["suite"]:
        sets.append("is_apartment = 1")
    if fb["boitier_pas_install"]:
        sets.append("box_not_installed = 1")

    # VLANs — individual columns
    if fb["vlan_manage"]:
        sets.append("vlan_manage = %(vlan_manage)s")
        params["vlan_manage"] = int(fb["vlan_manage"])
    if fb["vlan_internet"]:
        sets.append("vlan_internet = %(vlan_internet)s")
        params["vlan_internet"] = int(fb["vlan_internet"])
    if fb["vlan_telephone"]:
        sets.append("vlan_telephone = %(vlan_telephone)s")
        params["vlan_telephone"] = int(fb["vlan_telephone"])
    if fb["vlan_tele"]:
        sets.append("vlan_tv = %(vlan_tv)s")
        params["vlan_tv"] = int(fb["vlan_tele"])

    # Legacy service IDs
    for src, dst in [("manage_service_id", "manage_service_id"),
                     ("internet_service_id", "internet_service_id"),
                     ("telephone_service_id", "telephone_service_id"),
                     ("tele_service_id", "tv_service_id")]:
        if fb[src]:
            sets.append("{} = %({})s".format(dst, dst))
            params[dst] = int(fb[src])

    if fb["placemarks_id"]:
        sets.append("placemarks_id = %(placemarks_id)s")
        params["placemarks_id"] = str(fb["placemarks_id"])
    if fb["appartements_id"]:
        sets.append("apartment_building_id = %(apt_id)s")
        params["apt_id"] = str(fb["appartements_id"])

    if sets:
        sql = 'UPDATE "tabService Location" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated += 1

frappe.db.commit()
print("Updated {} locations with fibre detail".format(updated))

# Also import delivery.address2 and delivery.note
with conn.cursor() as cur:
    cur.execute("SELECT id, address2, note FROM delivery WHERE address2 IS NOT NULL OR note IS NOT NULL")
    deliveries = cur.fetchall()

updated_del = 0
for d in deliveries:
    loc_name = loc_map.get(d["id"])
    if not loc_name:
        continue
    sets = []
    params = {"name": loc_name}
    if d["address2"]:
        sets.append("address_line_2 = %(addr2)s")
        params["addr2"] = str(d["address2"]).strip()
    if d["note"]:
        sets.append("delivery_notes = %(note)s")
        params["note"] = str(d["note"])
    if sets:
        sql = 'UPDATE "tabService Location" SET {} WHERE name = %(name)s'.format(", ".join(sets))
        frappe.db.sql(sql, params)
        updated_del += 1

frappe.db.commit()
print("Updated {} locations with address2/notes".format(updated_del))


# ═══════════════════════════════════════════════════════════════
# 5. DONE
# ═══════════════════════════════════════════════════════════════
conn.close()
print("\n" + "=" * 60)
print("MIGRATION COMPLETE")
print("=" * 60)
print("Next: run migrate_provisioning_data.py for WiFi/VoIP from GenieACS")