607ea54b5c
Backend services: - targo-hub: extract deepGetValue to helpers.js, DRY disconnect reasons lookup map, compact CAPABILITIES, consolidate vision.js prompts/schemas, extract dispatch scoring weights, trim section dividers across 9 files - modem-bridge: extract getSession() helper (6 occurrences), resetIdleTimer(), consolidate DM query factory, fix duplicate username fill bug, trim headers (server.js -36%, tplink-session.js -47%, docker-compose.yml -57%) Frontend: - useWifiDiagnostic: extract THRESHOLDS const, split processDiagnostic into 6 focused helpers (processOnlineStatus, processWanIPs, processRadios, processMeshNodes, processClients, checkRadioIssues) - EquipmentDetail: merge duplicate ROLE_LABELS, remove verbose comments Documentation (17 → 13 files, -1,400 lines): - New consolidated README.md (architecture, services, dependencies, auth) - Merge ECOSYSTEM-OVERVIEW into ARCHITECTURE.md - Merge MIGRATION-PLAN + ARCHITECTURE-COMPARE + FIELD-GAP + CHANGELOG → MIGRATION.md - Merge COMPETITIVE-ANALYSIS into PLATFORM-STRATEGY.md - Update ROADMAP.md with current phase status - Delete CONTEXT.md (absorbed into README) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
369 lines
14 KiB
Python
369 lines
14 KiB
Python
"""
|
|
Import missing devices into Service Equipment and enrich Service Locations
|
|
with fibre data (connection_type, OLT port, VLANs).
|
|
|
|
Run inside erpnext-backend-1:
|
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_devices_and_enrich.py
|
|
"""
|
|
import frappe
|
|
import pymysql
|
|
import os
|
|
import time
|
|
import hashlib
|
|
from datetime import datetime
|
|
from decimal import Decimal
|
|
|
|
os.chdir("/home/frappe/frappe-bench/sites")
|
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
|
frappe.connect()
|
|
frappe.local.flags.ignore_permissions = True
|
|
print("Connected:", frappe.local.site)
|
|
|
|
T_TOTAL = time.time()
|
|
now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# LEGACY DB CONNECTION
|
|
# ═══════════════════════════════════════════════════════════════
|
|
conn = pymysql.connect(
|
|
host="legacy-db",
|
|
user="facturation",
|
|
password="*******",
|
|
database="gestionclient",
|
|
cursorclass=pymysql.cursors.DictCursor
|
|
)
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 1: Build lookup maps
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 1: BUILD LOOKUP MAPS")
|
|
print("="*60)
|
|
|
|
# Map legacy delivery_id → ERPNext Service Location name
|
|
loc_map = {}
|
|
rows = frappe.db.sql("""
|
|
SELECT name, legacy_delivery_id FROM "tabService Location"
|
|
WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id > 0
|
|
""", as_dict=True)
|
|
for r in rows:
|
|
loc_map[r["legacy_delivery_id"]] = r["name"]
|
|
print("Location map: {} entries".format(len(loc_map)))
|
|
|
|
# Map legacy account_id → ERPNext Customer name
|
|
cust_map = {}
|
|
rows = frappe.db.sql("""
|
|
SELECT name, legacy_account_id FROM "tabCustomer"
|
|
WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0
|
|
""", as_dict=True)
|
|
for r in rows:
|
|
cust_map[r["legacy_account_id"]] = r["name"]
|
|
print("Customer map: {} entries".format(len(cust_map)))
|
|
|
|
# Get already-imported device IDs
|
|
existing_devices = set()
|
|
rows = frappe.db.sql("""
|
|
SELECT legacy_device_id FROM "tabService Equipment"
|
|
WHERE legacy_device_id IS NOT NULL AND legacy_device_id > 0
|
|
""")
|
|
for r in rows:
|
|
existing_devices.add(r[0])
|
|
print("Already imported devices: {}".format(len(existing_devices)))
|
|
|
|
# Map delivery_id → account_id from legacy
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT id, account_id FROM delivery")
|
|
delivery_account = {}
|
|
for r in cur.fetchall():
|
|
delivery_account[r["id"]] = r["account_id"]
|
|
print("Delivery→account map: {} entries".format(len(delivery_account)))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 2: Import missing devices
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 2: IMPORT MISSING DEVICES")
|
|
print("="*60)
|
|
|
|
# Category mapping: legacy category → ERPNext equipment_type
|
|
CATEGORY_MAP = {
|
|
"onu": "ONT",
|
|
"tplink_tplg": "ONT",
|
|
"tplink_device2": "ONT",
|
|
"raisecom_rcmg": "ONT",
|
|
"stb": "Decodeur TV",
|
|
"stb_ministra": "Decodeur TV",
|
|
"airosm": "AP WiFi",
|
|
"airos_ac": "AP WiFi",
|
|
"cambium": "AP WiFi",
|
|
"ht803g1ge": "Telephone IP",
|
|
"custom": "Autre",
|
|
}
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT id, delivery_id, category, name, manufacturier, model,
|
|
sn, mac, manage, port, protocol, manage_cli, port_cli,
|
|
protocol_cli, user, pass, parent
|
|
FROM device ORDER BY id
|
|
""")
|
|
devices = cur.fetchall()
|
|
|
|
print("Total legacy devices: {}".format(len(devices)))
|
|
|
|
# Build set of existing serial numbers to avoid unique constraint violations
|
|
existing_serials = frappe.db.sql("""
|
|
SELECT serial_number FROM "tabService Equipment" WHERE serial_number IS NOT NULL
|
|
""")
|
|
seen_serials = set(r[0] for r in existing_serials)
|
|
print("Existing serial numbers: {}".format(len(seen_serials)))
|
|
|
|
inserted = 0
|
|
skipped = 0
|
|
batch = []
|
|
|
|
for dev in devices:
|
|
if dev["id"] in existing_devices:
|
|
skipped += 1
|
|
continue
|
|
|
|
equipment_type = CATEGORY_MAP.get(dev["category"], "Autre")
|
|
serial_number = dev["sn"] or "NO-SN-{}".format(dev["id"])
|
|
# Ensure uniqueness — append device ID if serial already seen
|
|
if serial_number in seen_serials:
|
|
serial_number = "{}-D{}".format(serial_number, dev["id"])
|
|
seen_serials.add(serial_number)
|
|
mac = dev["mac"] or None
|
|
brand = dev["manufacturier"] or None
|
|
model = dev["model"] or None
|
|
|
|
# Management IP — prefer manage_cli (clean IP), fallback to manage (may be URL)
|
|
ip_address = None
|
|
if dev["manage_cli"] and dev["manage_cli"].strip():
|
|
ip_address = dev["manage_cli"].strip()
|
|
elif dev["manage"] and dev["manage"].strip():
|
|
mgmt = dev["manage"].strip()
|
|
# If it's just an IP (not a full URL), use it
|
|
if not mgmt.startswith("http") and not "/" in mgmt:
|
|
ip_address = mgmt
|
|
|
|
login_user = dev["user"] if dev["user"] and dev["user"].strip() else None
|
|
login_pass = dev["pass"] if dev["pass"] and dev["pass"].strip() else None
|
|
|
|
# Link to Service Location via delivery_id
|
|
service_location = loc_map.get(dev["delivery_id"]) if dev["delivery_id"] else None
|
|
|
|
# Link to Customer via delivery → account
|
|
customer = None
|
|
if dev["delivery_id"] and dev["delivery_id"] in delivery_account:
|
|
account_id = delivery_account[dev["delivery_id"]]
|
|
customer = cust_map.get(account_id)
|
|
|
|
# Generate unique name
|
|
eq_name = "EQ-{}".format(hashlib.md5(str(dev["id"]).encode()).hexdigest()[:10])
|
|
|
|
batch.append({
|
|
"name": eq_name,
|
|
"now": now_str,
|
|
"equipment_type": equipment_type,
|
|
"serial_number": serial_number,
|
|
"mac_address": mac,
|
|
"brand": brand,
|
|
"model": model,
|
|
"ip_address": ip_address,
|
|
"login_user": login_user,
|
|
"login_password": login_pass,
|
|
"customer": customer,
|
|
"service_location": service_location,
|
|
"legacy_device_id": dev["id"],
|
|
"status": "Actif",
|
|
"ownership": "Gigafibre",
|
|
})
|
|
inserted += 1
|
|
|
|
# Insert in batches of 500
|
|
if len(batch) >= 500:
|
|
for eq in batch:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabService Equipment" (
|
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
|
equipment_type, serial_number, mac_address, brand, model,
|
|
ip_address, login_user, login_password,
|
|
customer, service_location, legacy_device_id,
|
|
status, ownership
|
|
) VALUES (
|
|
%(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, 0,
|
|
%(equipment_type)s, %(serial_number)s, %(mac_address)s, %(brand)s, %(model)s,
|
|
%(ip_address)s, %(login_user)s, %(login_password)s,
|
|
%(customer)s, %(service_location)s, %(legacy_device_id)s,
|
|
%(status)s, %(ownership)s
|
|
)
|
|
""", eq)
|
|
frappe.db.commit()
|
|
batch = []
|
|
print(" Inserted {}...".format(inserted))
|
|
|
|
# Final batch
|
|
if batch:
|
|
for eq in batch:
|
|
frappe.db.sql("""
|
|
INSERT INTO "tabService Equipment" (
|
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
|
equipment_type, serial_number, mac_address, brand, model,
|
|
ip_address, login_user, login_password,
|
|
customer, service_location, legacy_device_id,
|
|
status, ownership
|
|
) VALUES (
|
|
%(name)s, %(now)s, %(now)s, 'Administrator', 'Administrator', 0, 0,
|
|
%(equipment_type)s, %(serial_number)s, %(mac_address)s, %(brand)s, %(model)s,
|
|
%(ip_address)s, %(login_user)s, %(login_password)s,
|
|
%(customer)s, %(service_location)s, %(legacy_device_id)s,
|
|
%(status)s, %(ownership)s
|
|
)
|
|
""", eq)
|
|
frappe.db.commit()
|
|
|
|
print("Inserted {} new devices ({} already existed)".format(inserted, skipped))
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 3: Enrich Service Locations with fibre data
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 3: ENRICH LOCATIONS WITH FIBRE DATA")
|
|
print("="*60)
|
|
|
|
with conn.cursor() as cur:
|
|
# Get fibre data with service → delivery link (including OLT IP and name)
|
|
cur.execute("""
|
|
SELECT f.service_id, s.delivery_id,
|
|
f.frame, f.slot, f.port, f.ontid,
|
|
f.vlan_manage, f.vlan_internet, f.vlan_telephone, f.vlan_tele,
|
|
f.sn as ont_sn, f.tech as fibre_tech,
|
|
f.info_connect as olt_ip, fo.description as olt_name
|
|
FROM fibre f
|
|
LEFT JOIN fibre_olt fo ON f.info_connect = fo.ip
|
|
LEFT JOIN service s ON f.service_id = s.id
|
|
WHERE s.delivery_id IS NOT NULL
|
|
""")
|
|
fibre_data = cur.fetchall()
|
|
|
|
print("Fibre records with delivery link: {}".format(len(fibre_data)))
|
|
|
|
updated_locs = 0
|
|
for fb in fibre_data:
|
|
loc_name = loc_map.get(fb["delivery_id"])
|
|
if not loc_name:
|
|
continue
|
|
|
|
# Build OLT port string: "OLT_IP frame/slot/port ONT:id" or "frame/slot/port ONT:id"
|
|
olt_ip = fb.get("olt_ip") or ""
|
|
olt_name = fb.get("olt_name") or ""
|
|
frame = fb["frame"] if fb["frame"] is not None else 0
|
|
slot = fb["slot"] if fb["slot"] is not None else 0
|
|
port = fb["port"] if fb["port"] is not None else 0
|
|
|
|
olt_port_parts = []
|
|
if olt_ip:
|
|
olt_port_parts.append(str(olt_ip))
|
|
olt_port_parts.append("{}/{}/{}".format(frame, slot, port))
|
|
if fb["ontid"]:
|
|
olt_port_parts[-1] += " ONT:{}".format(fb["ontid"])
|
|
olt_port = " ".join(olt_port_parts)
|
|
|
|
vlans = []
|
|
if fb["vlan_internet"]:
|
|
vlans.append("inet:{}".format(fb["vlan_internet"]))
|
|
if fb["vlan_manage"]:
|
|
vlans.append("mgmt:{}".format(fb["vlan_manage"]))
|
|
if fb["vlan_telephone"]:
|
|
vlans.append("tel:{}".format(fb["vlan_telephone"]))
|
|
if fb["vlan_tele"]:
|
|
vlans.append("tv:{}".format(fb["vlan_tele"]))
|
|
network_id = " ".join(vlans) if vlans else None
|
|
|
|
# Build network_notes with OLT name if available
|
|
network_notes = "OLT: {}".format(olt_name) if olt_name else None
|
|
|
|
frappe.db.sql("""
|
|
UPDATE "tabService Location"
|
|
SET connection_type = 'Fibre FTTH',
|
|
olt_port = %(olt_port)s,
|
|
network_id = %(network_id)s,
|
|
network_notes = COALESCE(%(network_notes)s, network_notes)
|
|
WHERE name = %(name)s
|
|
AND (connection_type IS NULL OR connection_type = '' OR connection_type = 'Fibre FTTH')
|
|
""", {"name": loc_name, "olt_port": olt_port, "network_id": network_id, "network_notes": network_notes})
|
|
updated_locs += 1
|
|
|
|
frappe.db.commit()
|
|
print("Enriched {} locations with fibre data".format(updated_locs))
|
|
|
|
# Also set connection_type for locations with wireless devices
|
|
with conn.cursor() as cur:
|
|
cur.execute("""
|
|
SELECT DISTINCT d.delivery_id
|
|
FROM device d
|
|
WHERE d.category IN ('airosm', 'airos_ac', 'cambium')
|
|
AND d.delivery_id > 0
|
|
""")
|
|
wireless_deliveries = [r["delivery_id"] for r in cur.fetchall()]
|
|
|
|
wireless_updated = 0
|
|
for del_id in wireless_deliveries:
|
|
loc_name = loc_map.get(del_id)
|
|
if loc_name:
|
|
frappe.db.sql("""
|
|
UPDATE "tabService Location"
|
|
SET connection_type = 'Sans-fil'
|
|
WHERE name = %s
|
|
AND (connection_type IS NULL OR connection_type = '')
|
|
""", (loc_name,))
|
|
wireless_updated += 1
|
|
|
|
frappe.db.commit()
|
|
print("Set {} locations as Sans-fil (wireless)".format(wireless_updated))
|
|
|
|
conn.close()
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
# PHASE 4: VERIFY
|
|
# ═══════════════════════════════════════════════════════════════
|
|
print("\n" + "="*60)
|
|
print("PHASE 4: VERIFY")
|
|
print("="*60)
|
|
|
|
total_eq = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment"')[0][0]
|
|
by_type = frappe.db.sql("""
|
|
SELECT equipment_type, COUNT(*) as cnt FROM "tabService Equipment"
|
|
GROUP BY equipment_type ORDER BY cnt DESC
|
|
""", as_dict=True)
|
|
print("Total Service Equipment: {}".format(total_eq))
|
|
for t in by_type:
|
|
print(" {}: {}".format(t["equipment_type"], t["cnt"]))
|
|
|
|
with_customer = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment" WHERE customer IS NOT NULL')[0][0]
|
|
with_location = frappe.db.sql('SELECT COUNT(*) FROM "tabService Equipment" WHERE service_location IS NOT NULL')[0][0]
|
|
with_ip = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Equipment\" WHERE ip_address IS NOT NULL")[0][0]
|
|
with_creds = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Equipment\" WHERE login_user IS NOT NULL")[0][0]
|
|
print("\nWith customer link: {}".format(with_customer))
|
|
print("With service_location link: {}".format(with_location))
|
|
print("With IP address: {}".format(with_ip))
|
|
print("With credentials: {}".format(with_creds))
|
|
|
|
# Location enrichment stats
|
|
loc_by_conn = frappe.db.sql("""
|
|
SELECT connection_type, COUNT(*) as cnt FROM "tabService Location"
|
|
GROUP BY connection_type ORDER BY cnt DESC
|
|
""", as_dict=True)
|
|
print("\nService Locations by connection type:")
|
|
for l in loc_by_conn:
|
|
print(" {}: {}".format(l["connection_type"] or "(not set)", l["cnt"]))
|
|
|
|
with_olt = frappe.db.sql("SELECT COUNT(*) FROM \"tabService Location\" WHERE olt_port IS NOT NULL")[0][0]
|
|
print("With OLT port: {}".format(with_olt))
|
|
|
|
elapsed = time.time() - T_TOTAL
|
|
print("\n" + "="*60)
|
|
print("DONE in {:.1f}s".format(elapsed))
|
|
print("="*60)
|