gigafibre-fsm/scripts/migration/migrate_locations.py

#!/usr/bin/env python3
"""
Migrate legacy delivery → Service Location, device → Service Equipment.
Then link existing Subscriptions and Issues to their Service Location.

Dependencies: migrate_all.py must have run first (Customers, Subscriptions, Issues exist).

Run inside erpnext-backend-1:
  nohup python3 /tmp/migrate_locations.py > /tmp/migrate_locations.log 2>&1 &
  tail -f /tmp/migrate_locations.log

Phase 1: Add legacy_delivery_id custom field + column to Service Location
Phase 2: Import deliveries → Service Location
Phase 3: Import devices → Service Equipment
Phase 4: Link Subscriptions → Service Location (via legacy service.delivery_id)
Phase 5: Link Issues → Service Location (via legacy ticket.delivery_id)
"""
import pymysql
import psycopg2
import uuid
from datetime import datetime, timezone
from html import unescape

LEGACY = {"host": "legacy-db", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

ADMIN = "Administrator"

# Legacy device category → ERPNext equipment_type
DEVICE_TYPE_MAP = {
    "cpe": "ONT",
    "ont": "ONT",
    "onu": "ONT",
    "modem": "Modem",
    "routeur": "Routeur",
    "router": "Routeur",
    "switch": "Switch",
    "ap": "AP WiFi",
    "access point": "AP WiFi",
    "decodeur": "Decodeur TV",
    "stb": "Decodeur TV",
    "telephone": "Telephone IP",
    "ata": "Telephone IP",
    "amplificateur": "Amplificateur",
}

def uid(prefix=""):
    return prefix + uuid.uuid4().hex[:10]

def ts():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")

def clean(val):
    if not val:
        return ""
    return unescape(str(val)).strip()

def log(msg):
    print("[{}] {}".format(datetime.now(timezone.utc).strftime("%H:%M:%S"), msg), flush=True)

def guess_device_type(category, name, model):
    """Map legacy device category/name to ERPNext equipment_type."""
    cat = clean(category).lower()
    nm = clean(name).lower()
    mdl = clean(model).lower()
    combined = "{} {} {}".format(cat, nm, mdl)

    for key, val in DEVICE_TYPE_MAP.items():
        if key in combined:
            return val

    # Fallback heuristics
    if "fibre" in combined or "gpon" in combined:
        return "ONT"
    if "wifi" in combined or "wireless" in combined:
        return "AP WiFi"

    return "Autre"


def main():
    log("=" * 60)
    log("MIGRATE LOCATIONS + EQUIPMENT")
    log("=" * 60)

    mc = pymysql.connect(**LEGACY)
    pg = psycopg2.connect(**PG)
    pg.autocommit = False
    pgc = pg.cursor()
    now = ts()

    # ============================
    # Phase 1: Ensure legacy_delivery_id column exists
    # ============================
    log("")
    log("--- Phase 1: Ensure custom fields ---")

    pgc.execute("""SELECT column_name FROM information_schema.columns
                   WHERE table_name = 'tabService Location' AND column_name = 'legacy_delivery_id'""")
    if not pgc.fetchone():
        pgc.execute('ALTER TABLE "tabService Location" ADD COLUMN legacy_delivery_id bigint')
        # Also register as Custom Field so ERPNext knows about it
        try:
            pgc.execute("""
                INSERT INTO "tabCustom Field" (name, creation, modified, modified_by, owner, docstatus, idx,
                    dt, label, fieldname, fieldtype, insert_after)
                VALUES (%s, %s, %s, %s, %s, 0, 0,
                    'Service Location', 'Legacy Delivery ID', 'legacy_delivery_id', 'Int', 'access_notes')
            """, (uid("CF-"), now, now, ADMIN, ADMIN))
        except:
            pg.rollback()
        pg.commit()
        log("  Added legacy_delivery_id to Service Location")
    else:
        log("  legacy_delivery_id already exists")

    # Ensure legacy_device_id on Service Equipment
    pgc.execute("""SELECT column_name FROM information_schema.columns
                   WHERE table_name = 'tabService Equipment' AND column_name = 'legacy_device_id'""")
    if not pgc.fetchone():
        pgc.execute('ALTER TABLE "tabService Equipment" ADD COLUMN legacy_device_id bigint')
        try:
            pgc.execute("""
                INSERT INTO "tabCustom Field" (name, creation, modified, modified_by, owner, docstatus, idx,
                    dt, label, fieldname, fieldtype, insert_after)
                VALUES (%s, %s, %s, %s, %s, 0, 0,
                    'Service Equipment', 'Legacy Device ID', 'legacy_device_id', 'Int', 'notes')
            """, (uid("CF-"), now, now, ADMIN, ADMIN))
        except:
            pg.rollback()
        pg.commit()
        log("  Added legacy_device_id to Service Equipment")
    else:
        log("  legacy_device_id already exists")

    # ============================
    # Phase 2: Import deliveries → Service Location
    # ============================
    log("")
    log("=" * 60)
    log("Phase 2: Deliveries → Service Location")
    log("=" * 60)

    cur = mc.cursor(pymysql.cursors.DictCursor)
    cur.execute("SELECT * FROM delivery ORDER BY id")
    deliveries = cur.fetchall()
    log("  {} deliveries loaded".format(len(deliveries)))

    # Customer mapping
    pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0')
    cust_map = {r[0]: r[1] for r in pgc.fetchall()}

    # Check existing
    pgc.execute('SELECT legacy_delivery_id FROM "tabService Location" WHERE legacy_delivery_id > 0')
    existing_loc = set(r[0] for r in pgc.fetchall())
    log("  {} already imported".format(len(existing_loc)))

    # delivery_id → Service Location name mapping (for phases 3-5)
    del_map = {}
    loc_ok = loc_skip = loc_err = 0

    for i, d in enumerate(deliveries):
        did = d["id"]

        if did in existing_loc:
            # Still need the mapping for later phases
            loc_skip += 1
            continue

        cust_id = cust_map.get(d["account_id"])
        if not cust_id:
            loc_err += 1
            continue

        addr = clean(d.get("address1"))
        city = clean(d.get("city"))
        loc_name_display = clean(d.get("name")) or "{}, {}".format(addr, city) if addr else "Location-{}".format(did)
        loc_id = uid("LOC-")

        # Parse GPS
        lat = 0
        lon = 0
        try:
            if d.get("latitude"):
                lat = float(d["latitude"])
            if d.get("longitude"):
                lon = float(d["longitude"])
        except (ValueError, TypeError):
            pass

        try:
            pgc.execute("""
                INSERT INTO "tabService Location" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    customer, location_name, status,
                    address_line, city, postal_code, province,
                    latitude, longitude,
                    contact_name, contact_phone,
                    legacy_delivery_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    %s, %s, 'Active',
                    %s, %s, %s, %s,
                    %s, %s,
                    %s, %s,
                    %s
                )
            """, (loc_id, now, now, ADMIN, ADMIN,
                  cust_id, loc_name_display[:140],
                  addr or "N/A", city or "N/A",
                  clean(d.get("zip")) or None,
                  clean(d.get("state")) or "QC",
                  lat, lon,
                  clean(d.get("contact")) or None,
                  clean(d.get("tel_home")) or clean(d.get("cell")) or None,
                  did))

            del_map[did] = loc_id
            loc_ok += 1

        except Exception as e:
            loc_err += 1
            pg.rollback()
            if loc_err <= 10:
                log("  ERR del#{} -> {}".format(did, str(e)[:100]))
            continue

        if loc_ok % 1000 == 0:
            pg.commit()
            log("  [{}/{}] ok={} skip={} err={}".format(i+1, len(deliveries), loc_ok, loc_skip, loc_err))

    pg.commit()

    # Load mapping for skipped (already existing) locations
    if loc_skip > 0:
        pgc.execute('SELECT legacy_delivery_id, name FROM "tabService Location" WHERE legacy_delivery_id > 0')
        for lid, lname in pgc.fetchall():
            del_map[lid] = lname

    log("  Service Locations: {} created | {} skipped | {} errors".format(loc_ok, loc_skip, loc_err))
    log("  del_map has {} entries".format(len(del_map)))

    # ============================
    # Phase 3: Import devices → Service Equipment
    # ============================
    log("")
    log("=" * 60)
    log("Phase 3: Devices → Service Equipment")
    log("=" * 60)

    cur.execute("SELECT * FROM device ORDER BY id")
    devices = cur.fetchall()
    log("  {} devices loaded".format(len(devices)))

    pgc.execute('SELECT legacy_device_id FROM "tabService Equipment" WHERE legacy_device_id > 0')
    existing_dev = set(r[0] for r in pgc.fetchall())

    # device_id → Equipment name mapping (for parent hierarchy)
    dev_map = {}
    dev_ok = dev_skip = dev_err = 0

    for i, dv in enumerate(devices):
        dvid = dv["id"]

        if dvid in existing_dev:
            dev_skip += 1
            continue

        loc_id = del_map.get(dv.get("delivery_id"))
        # Get customer from the location's customer, or from delivery → account
        cust_id = None
        if loc_id:
            pgc.execute('SELECT customer FROM "tabService Location" WHERE name = %s', (loc_id,))
            row = pgc.fetchone()
            if row:
                cust_id = row[0]

        sn = (clean(dv.get("sn")) or "SN-{}".format(dvid))[:140]
        mac = clean(dv.get("mac"))[:140] if dv.get("mac") else None
        equip_type = guess_device_type(
            dv.get("category"), dv.get("name"), dv.get("model"))
        equip_id = uid("EQ-")

        try:
            pgc.execute("""
                INSERT INTO "tabService Equipment" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    equipment_type, brand, model, serial_number, mac_address,
                    customer, service_location, status, ownership,
                    ip_address, login_user, login_password,
                    legacy_device_id
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    %s, %s, %s, %s, %s,
                    %s, %s, 'Actif', 'Gigafibre',
                    %s, %s, %s,
                    %s
                )
            """, (equip_id, now, now, ADMIN, ADMIN,
                  equip_type,
                  clean(dv.get("manufacturier")) or None,
                  clean(dv.get("model")) or None,
                  sn[:140],
                  mac or None,
                  cust_id,
                  loc_id,
                  clean(dv.get("manage")) or None,
                  clean(dv.get("user")) or None,
                  clean(dv.get("pass")) or None,
                  dvid))

            dev_map[dvid] = equip_id
            dev_ok += 1

        except Exception as e:
            pg.rollback()
            # Retry with unique SN on duplicate key
            if "unique constraint" in str(e).lower() and "serial_number" in str(e).lower():
                sn = "{}-{}".format(sn[:130], dvid)
                try:
                    pgc.execute("""
                        INSERT INTO "tabService Equipment" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            equipment_type, brand, model, serial_number, mac_address,
                            customer, service_location, status, ownership,
                            ip_address, login_user, login_password,
                            legacy_device_id
                        ) VALUES (
                            %s, %s, %s, %s, %s, 0, 0,
                            %s, %s, %s, %s, %s,
                            %s, %s, 'Actif', 'Gigafibre',
                            %s, %s, %s,
                            %s
                        )
                    """, (equip_id, now, now, ADMIN, ADMIN,
                          equip_type,
                          clean(dv.get("manufacturier")) or None,
                          clean(dv.get("model")) or None,
                          sn, mac,
                          cust_id, loc_id,
                          clean(dv.get("manage")) or None,
                          clean(dv.get("user")) or None,
                          clean(dv.get("pass")) or None,
                          dvid))
                    dev_map[dvid] = equip_id
                    dev_ok += 1
                    continue
                except Exception as e2:
                    pg.rollback()

            dev_err += 1
            if dev_err <= 10:
                log("  ERR dev#{} -> {}".format(dvid, str(e)[:100]))
            continue

        if dev_ok % 1000 == 0:
            pg.commit()
            log("  [{}/{}] ok={} skip={} err={}".format(i+1, len(devices), dev_ok, dev_skip, dev_err))

    pg.commit()
    log("  Equipment: {} created | {} skipped | {} errors".format(dev_ok, dev_skip, dev_err))

    # Phase 3b: Set parent equipment (device hierarchy)
    log("  Setting device parent hierarchy...")
    parent_set = 0
    for dv in devices:
        if dv.get("parent") and dv["parent"] > 0:
            child_eq = dev_map.get(dv["id"])
            parent_eq = dev_map.get(dv["parent"])
            if child_eq and parent_eq:
                # No native parent field on Service Equipment, store in notes for now
                pgc.execute("""
                    UPDATE "tabService Equipment"
                    SET notes = COALESCE(notes, '') || 'Parent: ' || %s || E'\n'
                    WHERE name = %s
                """, (parent_eq, child_eq))
                parent_set += 1
    pg.commit()
    log("  {} parent links set".format(parent_set))

    # ============================
    # Phase 4: Link Subscriptions → Service Location
    # ============================
    log("")
    log("=" * 60)
    log("Phase 4: Link Subscriptions → Service Location")
    log("=" * 60)

    # Get service → delivery mapping from legacy
    cur.execute("SELECT id, delivery_id FROM service WHERE status = 1 AND delivery_id > 0")
    svc_to_del = {r["id"]: r["delivery_id"] for r in cur.fetchall()}
    log("  {} service→delivery mappings".format(len(svc_to_del)))

    # Get subscriptions with legacy_service_id
    pgc.execute("""
        SELECT name, legacy_service_id FROM "tabSubscription"
        WHERE legacy_service_id > 0
        AND (service_location IS NULL OR service_location = '')
    """)
    subs_to_link = pgc.fetchall()
    log("  {} subscriptions to link".format(len(subs_to_link)))

    sub_linked = sub_miss = 0
    for sub_name, legacy_svc_id in subs_to_link:
        del_id = svc_to_del.get(legacy_svc_id)
        if not del_id:
            sub_miss += 1
            continue
        loc_id = del_map.get(del_id)
        if not loc_id:
            sub_miss += 1
            continue

        pgc.execute("""
            UPDATE "tabSubscription"
            SET service_location = %s, modified = NOW()
            WHERE name = %s
        """, (loc_id, sub_name))
        sub_linked += 1

        if sub_linked % 5000 == 0:
            pg.commit()
            log("  {} linked...".format(sub_linked))

    pg.commit()
    log("  Subscriptions linked: {} | missed: {}".format(sub_linked, sub_miss))

    # ============================
    # Phase 5: Link Issues → Service Location
    # ============================
    log("")
    log("=" * 60)
    log("Phase 5: Link Issues → Service Location")
    log("=" * 60)

    # Get ticket → delivery mapping from legacy
    cur.execute("SELECT id, delivery_id FROM ticket WHERE delivery_id > 0")
    tkt_to_del = {r["id"]: r["delivery_id"] for r in cur.fetchall()}
    log("  {} ticket→delivery mappings".format(len(tkt_to_del)))

    # Get issues with legacy_ticket_id that need linking
    pgc.execute("""
        SELECT name, legacy_ticket_id FROM "tabIssue"
        WHERE legacy_ticket_id > 0
        AND (service_location IS NULL OR service_location = '')
    """)
    issues_to_link = pgc.fetchall()
    log("  {} issues to link".format(len(issues_to_link)))

    iss_linked = iss_miss = 0
    for issue_name, legacy_tkt_id in issues_to_link:
        del_id = tkt_to_del.get(legacy_tkt_id)
        if not del_id:
            iss_miss += 1
            continue
        loc_id = del_map.get(del_id)
        if not loc_id:
            iss_miss += 1
            continue

        pgc.execute("""
            UPDATE "tabIssue"
            SET service_location = %s, modified = NOW()
            WHERE name = %s
        """, (loc_id, issue_name))
        iss_linked += 1

        if iss_linked % 10000 == 0:
            pg.commit()
            log("  {} linked...".format(iss_linked))

    pg.commit()
    log("  Issues linked: {} | missed: {}".format(iss_linked, iss_miss))

    # ============================
    # Summary
    # ============================
    mc.close()
    pg.close()

    log("")
    log("=" * 60)
    log("MIGRATION LOCATIONS + EQUIPMENT COMPLETE")
    log("=" * 60)
    log("  Service Locations: {} created".format(loc_ok))
    log("  Service Equipment: {} created ({} parent links)".format(dev_ok, parent_set))
    log("  Subscriptions → Location: {} linked".format(sub_linked))
    log("  Issues → Location: {} linked".format(iss_linked))
    log("=" * 60)
    log("")
    log("Next: bench --site erp.gigafibre.ca clear-cache")

if __name__ == "__main__":
    main()