320655b0a0
- Remove apps/dispatch/ (100% replaced by ops dispatch module, unmaintained) - Commit services/targo-hub/lib/ (24 modules, 6290 lines — was never tracked) - Commit services/docuseal + services/legacy-db docker-compose configs - Extract client app composables: useOTP, useAddressSearch, catalog data, format utils - Refactor CartPage.vue 630→175 lines, CatalogPage.vue 375→95 lines - Clean hardcoded credentials from config.js fallback values - Add client portal: catalog, cart, checkout, OTP verification, address search - Add ops: NetworkPage, AgentFlowsPage, ConversationPanel, UnifiedCreateModal - Add ops composables: useBestTech, useConversations, usePermissions, useScanner - Add field app: scanner composable, docker/nginx configs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
516 lines
19 KiB
Python
516 lines
19 KiB
Python
"""
|
|
migrate_missing_data.py — Populate the new custom fields from legacy DB.
|
|
|
|
Prerequisites: add_missing_custom_fields.py must have been run first.
|
|
|
|
Migrates:
|
|
1. Customer: PPA, Stripe, VIP, termination, portal, flags
|
|
2. Service Subscription: hijack details, dates, quotas
|
|
3. Service Equipment: management access, parent hierarchy, legacy category
|
|
4. Service Location: fibre detail (VLANs individual, OLT IP/name, ONT ID, etc.)
|
|
5. Dispatch Job: tech times from bon_travail
|
|
|
|
Run inside erpnext-backend-1:
|
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/migrate_missing_data.py
|
|
"""
|
|
import frappe
|
|
import pymysql
|
|
import os
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
|
|
os.chdir("/home/frappe/frappe-bench/sites")
|
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
|
frappe.connect()
|
|
frappe.local.flags.ignore_permissions = True
|
|
print("Connected:", frappe.local.site)
|
|
|
|
conn = pymysql.connect(
|
|
host="10.100.80.100",
|
|
user="facturation",
|
|
password="VD67owoj",
|
|
database="gestionclient",
|
|
cursorclass=pymysql.cursors.DictCursor
|
|
)
|
|
|
|
def ts_to_date(ts):
|
|
"""Convert unix timestamp to YYYY-MM-DD, return None on failure."""
|
|
if not ts or ts == 0:
|
|
return None
|
|
try:
|
|
return datetime.fromtimestamp(int(ts)).strftime("%Y-%m-%d")
|
|
except (ValueError, OSError, OverflowError):
|
|
return None
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# BUILD MAPS
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("Building lookup maps...")
|
|
|
|
# legacy_account_id → ERPNext Customer name
|
|
cust_map = {}
|
|
for r in frappe.db.sql('SELECT name, legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0', as_dict=True):
|
|
cust_map[r["legacy_account_id"]] = r["name"]
|
|
print(" Customer map: {}".format(len(cust_map)))
|
|
|
|
# legacy_delivery_id → ERPNext Service Location name
|
|
loc_map = {}
|
|
for r in frappe.db.sql('SELECT name, legacy_delivery_id FROM "tabService Location" WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id > 0', as_dict=True):
|
|
loc_map[r["legacy_delivery_id"]] = r["name"]
|
|
print(" Location map: {}".format(len(loc_map)))
|
|
|
|
# legacy_service_id → ERPNext Service Subscription name
|
|
sub_map = {}
|
|
for r in frappe.db.sql('SELECT name, legacy_service_id FROM "tabService Subscription" WHERE legacy_service_id IS NOT NULL AND legacy_service_id > 0', as_dict=True):
|
|
sub_map[r["legacy_service_id"]] = r["name"]
|
|
print(" Subscription map: {}".format(len(sub_map)))
|
|
|
|
# legacy_device_id → ERPNext Service Equipment name
|
|
dev_map = {}
|
|
for r in frappe.db.sql('SELECT name, legacy_device_id FROM "tabService Equipment" WHERE legacy_device_id IS NOT NULL AND legacy_device_id > 0', as_dict=True):
|
|
dev_map[r["legacy_device_id"]] = r["name"]
|
|
print(" Device map: {}".format(len(dev_map)))
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# 1. CUSTOMER — PPA, Stripe, VIP, termination, portal, flags
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "=" * 60)
|
|
print("1. CUSTOMER — PPA, Stripe, VIP, termination, portal")
|
|
print("=" * 60)
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT id, middle_name, title,
|
|
ppa, ppa_name, ppa_code, ppa_branch, ppa_account,
|
|
ppa_amount, ppa_amount_buffer, ppa_fixed, ppa_cc, ppa_all_invoice,
|
|
stripe_id, stripe_ppa,
|
|
vip, land_owner, pub, `call`,
|
|
terminate_reason, terminate_cie, terminate_note, terminate_date,
|
|
notes_client,
|
|
username, password, keyword
|
|
FROM account
|
|
""")
|
|
accounts = cur.fetchall()
|
|
|
|
updated = 0
|
|
for acct in accounts:
|
|
cust_name = cust_map.get(acct["id"])
|
|
if not cust_name:
|
|
continue
|
|
|
|
sets = []
|
|
params = {"name": cust_name}
|
|
|
|
# PPA fields
|
|
if acct["ppa"]:
|
|
sets.append("ppa_enabled = 1")
|
|
for f in ["ppa_name", "ppa_code", "ppa_branch", "ppa_account"]:
|
|
if acct.get(f):
|
|
sets.append("{} = %({})s".format(f, f))
|
|
params[f] = str(acct[f]).strip() if acct[f] else None
|
|
if acct["ppa_amount"]:
|
|
sets.append("ppa_amount = %(ppa_amount)s")
|
|
params["ppa_amount"] = float(acct["ppa_amount"])
|
|
if acct["ppa_amount_buffer"]:
|
|
sets.append("ppa_amount_buffer = %(ppa_amount_buffer)s")
|
|
params["ppa_amount_buffer"] = float(acct["ppa_amount_buffer"])
|
|
if acct["ppa_fixed"]:
|
|
sets.append("ppa_fixed = 1")
|
|
if acct["ppa_cc"]:
|
|
sets.append("ppa_cc = 1")
|
|
if acct["ppa_all_invoice"]:
|
|
sets.append("ppa_all_invoice = 1")
|
|
|
|
# Stripe
|
|
if acct["stripe_id"]:
|
|
sets.append("stripe_customer_id = %(stripe_id)s")
|
|
params["stripe_id"] = str(acct["stripe_id"]).strip()
|
|
if acct["stripe_ppa"]:
|
|
sets.append("stripe_ppa_enabled = 1")
|
|
|
|
# Flags
|
|
if acct["vip"]:
|
|
sets.append("is_vip = 1")
|
|
if acct["land_owner"]:
|
|
sets.append("is_land_owner = 1")
|
|
if acct["pub"]:
|
|
sets.append("marketing_optin = 1")
|
|
if acct.get("call"):
|
|
sets.append("call_contact = 1")
|
|
|
|
# Termination
|
|
if acct["terminate_reason"]:
|
|
sets.append("terminate_reason = %(terminate_reason)s")
|
|
params["terminate_reason"] = str(acct["terminate_reason"])
|
|
if acct["terminate_cie"]:
|
|
sets.append("terminate_cie = %(terminate_cie)s")
|
|
params["terminate_cie"] = str(acct["terminate_cie"])
|
|
if acct["terminate_note"]:
|
|
sets.append("terminate_note = %(terminate_note)s")
|
|
params["terminate_note"] = str(acct["terminate_note"])
|
|
if acct["terminate_date"]:
|
|
td = ts_to_date(acct["terminate_date"])
|
|
if td:
|
|
sets.append("terminate_date = %(terminate_date)s")
|
|
params["terminate_date"] = td
|
|
|
|
# Client notes
|
|
if acct["notes_client"]:
|
|
sets.append("notes_client = %(notes_client)s")
|
|
params["notes_client"] = str(acct["notes_client"])
|
|
|
|
# Portal
|
|
if acct["username"]:
|
|
sets.append("portal_username = %(portal_username)s")
|
|
params["portal_username"] = str(acct["username"])
|
|
if acct["password"]:
|
|
sets.append("portal_password_hash = %(portal_password_hash)s")
|
|
params["portal_password_hash"] = str(acct["password"])
|
|
|
|
# Misc
|
|
if acct["middle_name"]:
|
|
sets.append("middle_name = %(middle_name)s")
|
|
params["middle_name"] = str(acct["middle_name"]).strip()
|
|
if acct["title"]:
|
|
sets.append("salutation_legacy = %(salutation_legacy)s")
|
|
params["salutation_legacy"] = str(acct["title"]).strip()
|
|
if acct["keyword"]:
|
|
sets.append("search_keyword = %(search_keyword)s")
|
|
params["search_keyword"] = str(acct["keyword"]).strip()
|
|
|
|
if sets:
|
|
sql = 'UPDATE "tabCustomer" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} customers".format(updated))
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# 2. SERVICE SUBSCRIPTION — hijack details, dates, quotas
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "=" * 60)
|
|
print("2. SERVICE SUBSCRIPTION — hijack, dates, IP")
|
|
print("=" * 60)
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT id, hijack, hijack_desc,
|
|
hijack_quota_day, hijack_quota_night,
|
|
date_suspended, actif_until, date_next_invoice,
|
|
forfait_internet, radius_conso, ip_fixe
|
|
FROM service
|
|
""")
|
|
services = cur.fetchall()
|
|
|
|
updated = 0
|
|
for svc in services:
|
|
sub_name = sub_map.get(svc["id"])
|
|
if not sub_name:
|
|
continue
|
|
|
|
sets = []
|
|
params = {"name": sub_name}
|
|
|
|
if svc["hijack"]:
|
|
sets.append("is_custom_pricing = 1")
|
|
if svc["hijack_desc"]:
|
|
sets.append("custom_pricing_desc = %(hijack_desc)s")
|
|
params["hijack_desc"] = str(svc["hijack_desc"])
|
|
if svc["hijack_quota_day"]:
|
|
sets.append("quota_day_gb = %(quota_day)s")
|
|
params["quota_day"] = float(svc["hijack_quota_day"])
|
|
if svc["hijack_quota_night"]:
|
|
sets.append("quota_night_gb = %(quota_night)s")
|
|
params["quota_night"] = float(svc["hijack_quota_night"])
|
|
|
|
ds = ts_to_date(svc["date_suspended"])
|
|
if ds:
|
|
sets.append("date_suspended = %(date_suspended)s")
|
|
params["date_suspended"] = ds
|
|
au = ts_to_date(svc["actif_until"])
|
|
if au:
|
|
sets.append("active_until = %(active_until)s")
|
|
params["active_until"] = au
|
|
ni = ts_to_date(svc["date_next_invoice"])
|
|
if ni:
|
|
sets.append("next_invoice_date = %(next_invoice_date)s")
|
|
params["next_invoice_date"] = ni
|
|
|
|
if svc["forfait_internet"]:
|
|
sets.append("forfait_internet = 1")
|
|
if svc["radius_conso"]:
|
|
sets.append("radius_consumption = %(radius_conso)s")
|
|
params["radius_conso"] = str(svc["radius_conso"])
|
|
if svc.get("ip_fixe"):
|
|
sets.append("static_ip = %(static_ip)s")
|
|
params["static_ip"] = str(svc["ip_fixe"])
|
|
|
|
if sets:
|
|
sql = 'UPDATE "tabService Subscription" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} subscriptions".format(updated))
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# 3. SERVICE EQUIPMENT — management, parent, category, device_attr
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "=" * 60)
|
|
print("3. SERVICE EQUIPMENT — management, parent, category")
|
|
print("=" * 60)
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT id, manage, port, protocol, manage_cli, port_cli, protocol_cli,
|
|
parent, category, name as device_name
|
|
FROM device
|
|
""")
|
|
devices = cur.fetchall()
|
|
|
|
updated = 0
|
|
for dev in devices:
|
|
eq_name = dev_map.get(dev["id"])
|
|
if not eq_name:
|
|
continue
|
|
|
|
sets = []
|
|
params = {"name": eq_name}
|
|
|
|
if dev["manage"]:
|
|
sets.append("manage_url = %(manage_url)s")
|
|
params["manage_url"] = str(dev["manage"]).strip()
|
|
if dev["port"]:
|
|
sets.append("manage_port = %(manage_port)s")
|
|
params["manage_port"] = int(dev["port"])
|
|
if dev["protocol"]:
|
|
proto = str(dev["protocol"]).strip().upper()
|
|
if proto in ("HTTP", "HTTPS", "SSH", "TELNET", "SNMP"):
|
|
sets.append("manage_protocol = %(manage_protocol)s")
|
|
params["manage_protocol"] = proto
|
|
if dev["manage_cli"]:
|
|
sets.append("cli_ip = %(cli_ip)s")
|
|
params["cli_ip"] = str(dev["manage_cli"]).strip()
|
|
if dev["port_cli"]:
|
|
sets.append("cli_port = %(cli_port)s")
|
|
params["cli_port"] = int(dev["port_cli"])
|
|
if dev["protocol_cli"]:
|
|
proto_cli = str(dev["protocol_cli"]).strip().upper()
|
|
if proto_cli in ("SSH", "TELNET"):
|
|
sets.append("cli_protocol = %(cli_protocol)s")
|
|
params["cli_protocol"] = proto_cli
|
|
if dev["parent"]:
|
|
parent_eq = dev_map.get(dev["parent"])
|
|
if parent_eq:
|
|
sets.append("parent_device = %(parent_device)s")
|
|
params["parent_device"] = parent_eq
|
|
if dev["category"]:
|
|
sets.append("legacy_category = %(legacy_category)s")
|
|
params["legacy_category"] = str(dev["category"])
|
|
if dev["device_name"]:
|
|
sets.append("device_name_legacy = %(device_name_legacy)s")
|
|
params["device_name_legacy"] = str(dev["device_name"])
|
|
|
|
if sets:
|
|
sql = 'UPDATE "tabService Equipment" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} devices".format(updated))
|
|
|
|
# ── device_attr → JSON MAC addresses + stb_id ──
|
|
print("\nImporting device_attr...")
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT device_id, `key`, value FROM device_attr ORDER BY device_id")
|
|
attrs = cur.fetchall()
|
|
|
|
# Group by device_id
|
|
device_attrs = {}
|
|
for attr in attrs:
|
|
did = attr["device_id"]
|
|
if did not in device_attrs:
|
|
device_attrs[did] = {}
|
|
device_attrs[did][attr["key"]] = attr["value"]
|
|
|
|
updated_attrs = 0
|
|
for did, kv in device_attrs.items():
|
|
eq_name = dev_map.get(did)
|
|
if not eq_name:
|
|
continue
|
|
|
|
sets = []
|
|
params = {"name": eq_name}
|
|
|
|
# Extract MAC addresses
|
|
macs = {}
|
|
stb_id = None
|
|
for k, v in kv.items():
|
|
if k.startswith("mac_") or k.startswith("eth"):
|
|
macs[k] = v
|
|
if k == "stb_id":
|
|
stb_id = v
|
|
|
|
if macs:
|
|
sets.append("mac_addresses_json = %(macs_json)s")
|
|
params["macs_json"] = json.dumps(macs)
|
|
if stb_id:
|
|
sets.append("iptv_subscription_id = %(stb_id)s")
|
|
params["stb_id"] = str(stb_id)
|
|
|
|
if sets:
|
|
sql = 'UPDATE "tabService Equipment" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated_attrs += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} devices with attrs".format(updated_attrs))
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# 4. SERVICE LOCATION — fibre detail (VLANs, OLT IP, ONT, etc.)
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "=" * 60)
|
|
print("4. SERVICE LOCATION — fibre detail, VLANs, building")
|
|
print("=" * 60)
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT f.id as fibre_id, f.service_id, f.sn as ont_sn,
|
|
f.info_connect as olt_ip, f.ontid, f.tech as terrain,
|
|
f.distance, f.nb_portees, f.temps_estim, f.suite,
|
|
f.boitier_pas_install,
|
|
f.vlan_manage, f.vlan_internet, f.vlan_telephone, f.vlan_tele,
|
|
f.manage_service_id, f.internet_service_id,
|
|
f.telephone_service_id, f.tele_service_id,
|
|
f.placemarks_id, f.appartements_id,
|
|
fo.description as olt_name,
|
|
s.delivery_id
|
|
FROM fibre f
|
|
LEFT JOIN fibre_olt fo ON f.info_connect = fo.ip
|
|
LEFT JOIN service s ON f.service_id = s.id
|
|
WHERE s.delivery_id IS NOT NULL
|
|
""")
|
|
fibres = cur.fetchall()
|
|
|
|
updated = 0
|
|
for fb in fibres:
|
|
loc_name = loc_map.get(fb["delivery_id"])
|
|
if not loc_name:
|
|
continue
|
|
|
|
sets = []
|
|
params = {"name": loc_name}
|
|
|
|
if fb["fibre_id"]:
|
|
sets.append("legacy_fibre_id = %(fibre_id)s")
|
|
params["fibre_id"] = fb["fibre_id"]
|
|
if fb["ont_sn"]:
|
|
sets.append("ont_serial = %(ont_sn)s")
|
|
params["ont_sn"] = str(fb["ont_sn"])
|
|
if fb["olt_ip"]:
|
|
sets.append("olt_ip = %(olt_ip)s")
|
|
params["olt_ip"] = str(fb["olt_ip"])
|
|
if fb["olt_name"]:
|
|
sets.append("olt_name = %(olt_name)s")
|
|
params["olt_name"] = str(fb["olt_name"])
|
|
if fb["ontid"]:
|
|
sets.append("ont_id = %(ontid)s")
|
|
params["ontid"] = int(fb["ontid"])
|
|
if fb["terrain"]:
|
|
sets.append("terrain_type = %(terrain)s")
|
|
params["terrain"] = str(fb["terrain"])
|
|
if fb["distance"]:
|
|
sets.append("fibre_distance_m = %(distance)s")
|
|
params["distance"] = float(fb["distance"])
|
|
if fb["nb_portees"]:
|
|
sets.append("fibre_spans = %(spans)s")
|
|
params["spans"] = int(fb["nb_portees"])
|
|
if fb["temps_estim"]:
|
|
sets.append("install_time_estimate = %(temps)s")
|
|
params["temps"] = str(fb["temps_estim"])
|
|
if fb["suite"]:
|
|
sets.append("is_apartment = 1")
|
|
if fb["boitier_pas_install"]:
|
|
sets.append("box_not_installed = 1")
|
|
|
|
# VLANs — individual columns
|
|
if fb["vlan_manage"]:
|
|
sets.append("vlan_manage = %(vlan_manage)s")
|
|
params["vlan_manage"] = int(fb["vlan_manage"])
|
|
if fb["vlan_internet"]:
|
|
sets.append("vlan_internet = %(vlan_internet)s")
|
|
params["vlan_internet"] = int(fb["vlan_internet"])
|
|
if fb["vlan_telephone"]:
|
|
sets.append("vlan_telephone = %(vlan_telephone)s")
|
|
params["vlan_telephone"] = int(fb["vlan_telephone"])
|
|
if fb["vlan_tele"]:
|
|
sets.append("vlan_tv = %(vlan_tv)s")
|
|
params["vlan_tv"] = int(fb["vlan_tele"])
|
|
|
|
# Legacy service IDs
|
|
for src, dst in [("manage_service_id", "manage_service_id"),
|
|
("internet_service_id", "internet_service_id"),
|
|
("telephone_service_id", "telephone_service_id"),
|
|
("tele_service_id", "tv_service_id")]:
|
|
if fb[src]:
|
|
sets.append("{} = %({})s".format(dst, dst))
|
|
params[dst] = int(fb[src])
|
|
|
|
if fb["placemarks_id"]:
|
|
sets.append("placemarks_id = %(placemarks_id)s")
|
|
params["placemarks_id"] = str(fb["placemarks_id"])
|
|
if fb["appartements_id"]:
|
|
sets.append("apartment_building_id = %(apt_id)s")
|
|
params["apt_id"] = str(fb["appartements_id"])
|
|
|
|
if sets:
|
|
sql = 'UPDATE "tabService Location" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} locations with fibre detail".format(updated))
|
|
|
|
# Also import delivery.address2 and delivery.note
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT id, address2, note FROM delivery WHERE address2 IS NOT NULL OR note IS NOT NULL")
|
|
deliveries = cur.fetchall()
|
|
|
|
updated_del = 0
|
|
for d in deliveries:
|
|
loc_name = loc_map.get(d["id"])
|
|
if not loc_name:
|
|
continue
|
|
sets = []
|
|
params = {"name": loc_name}
|
|
if d["address2"]:
|
|
sets.append("address_line_2 = %(addr2)s")
|
|
params["addr2"] = str(d["address2"]).strip()
|
|
if d["note"]:
|
|
sets.append("delivery_notes = %(note)s")
|
|
params["note"] = str(d["note"])
|
|
if sets:
|
|
sql = 'UPDATE "tabService Location" SET {} WHERE name = %(name)s'.format(", ".join(sets))
|
|
frappe.db.sql(sql, params)
|
|
updated_del += 1
|
|
|
|
frappe.db.commit()
|
|
print("Updated {} locations with address2/notes".format(updated_del))
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# 5. DONE
|
|
# ═══════════════════════════════════════════════════════════════
|
|
conn.close()
|
|
print("\n" + "=" * 60)
|
|
print("MIGRATION COMPLETE")
|
|
print("=" * 60)
|
|
print("Next: run migrate_provisioning_data.py for WiFi/VoIP from GenieACS")
|