louis/gigafibre-fsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
fd326ac52e
gigafibre-fsm/scripts/migration/migrate_direct.py
louispaulb 93dd7a525f feat: migration legacy → ERPNext phases 1-4 complete 
Phase 1: 833 Items + 34 Item Groups + custom fields (ISP speeds, RADIUS, legacy IDs)
Phase 2: 6,667 Customers + Contacts + Addresses via direct PG (~30s)
Phase 3: Tax template QC TPS+TVQ + 92 Subscription Plans
Phase 4: 21,876 Subscriptions with RADIUS data

CRITICAL: ERPNext scheduler is PAUSED — do not reactivate without explicit go.

Includes:
- ARCHITECTURE-COMPARE.md: full schema mapping legacy vs ERPNext
- CHANGELOG.md: detailed migration log
- MIGRATION-PLAN.md: strategy and next steps
- scripts/migration/: idempotent Python scripts (direct PG method)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-28 14:35:02 -04:00

230 lines
9.3 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
Direct PostgreSQL migration: Legacy account → ERPNext Customer + Contact + Address.
Writes directly to tabCustomer, tabContact, tabAddress, tabDynamic Link, etc.
No Frappe ORM, no HTTP API — fastest possible method.
Run inside erpnext-backend-1:
python3 /tmp/migrate_direct.py
Idempotent: skips customers where legacy_account_id already exists.
"""
import pymysql
import psycopg2
import uuid
from html import unescape
from datetime import datetime
# Legacy MariaDB
LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
"database": "gestionclient", "connect_timeout": 30, "read_timeout": 300}
# ERPNext PostgreSQL
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
"dbname": "_eb65bdc0c4b1b2d6"}
GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual",
7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"}
ADMIN = "Administrator"
def clean(val):
if not val:
return ""
return unescape(str(val)).strip()
def uid(prefix=""):
return prefix + uuid.uuid4().hex[:10]
def now():
return datetime.utcnow()
def main():
print("=== Direct PostgreSQL Migration v5 ===")
# 1. Read all legacy data in one shot
print("Reading legacy MariaDB...", flush=True)
mc = pymysql.connect(**LEGACY)
cur = mc.cursor(pymysql.cursors.DictCursor)
cur.execute("SELECT * FROM account WHERE status IN (1,2) ORDER BY id")
accounts = cur.fetchall()
cur.execute("SELECT * FROM delivery ORDER BY account_id")
all_del = cur.fetchall()
mc.close()
print(" {} accounts, {} deliveries loaded. MariaDB closed.".format(len(accounts), len(all_del)), flush=True)
# Delivery lookup
active_ids = set(a["id"] for a in accounts)
del_by = {}
for d in all_del:
if d["account_id"] in active_ids:
del_by.setdefault(d["account_id"], []).append(d)
# 2. Connect ERPNext PostgreSQL
print("Connecting to ERPNext PostgreSQL...", flush=True)
pg = psycopg2.connect(**PG)
pgc = pg.cursor()
# Existing customers
pgc.execute('SELECT legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0')
existing = set(r[0] for r in pgc.fetchall())
print(" {} already imported, will skip".format(len(existing)), flush=True)
ts = now()
c_ok = c_addr = c_contact = c_skip = c_err = 0
for i, a in enumerate(accounts):
aid = a["id"]
if aid in existing:
c_skip += 1
continue
first = clean(a["first_name"])
last = clean(a["last_name"])
company = clean(a["company"])
if company:
ctype, cname = "Company", company
else:
ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid)
cust_id = uid("CUST-")
group = GROUP_MAP.get(a["group_id"], "Individual")
lang = "fr" if a.get("language_id") == "francais" else "en"
disabled = 0 if a["status"] == 1 else 1
try:
# --- Customer ---
pgc.execute("""
INSERT INTO "tabCustomer" (
name, creation, modified, modified_by, owner, docstatus, idx,
naming_series, customer_name, customer_type, customer_group,
territory, default_currency, language, disabled,
legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id
) VALUES (
%s, %s, %s, %s, %s, 0, 0,
'CUST-.YYYY.-', %s, %s, %s,
'Canada', 'CAD', %s, %s,
%s, %s, %s, %s
)
""", (cust_id, ts, ts, ADMIN, ADMIN,
cname, ctype, group, lang, disabled,
aid, clean(a.get("customer_id")),
1 if a.get("ppa") else 0,
clean(a.get("stripe_id")) or None))
c_ok += 1
# --- Contact ---
email = clean(a.get("email"))
tel = clean(a.get("tel_home"))
cell = clean(a.get("cell"))
if first or email:
cont_id = uid("CONT-")
full = "{} {}".format(first, last).strip()
pgc.execute("""
INSERT INTO "tabContact" (
name, creation, modified, modified_by, owner, docstatus, idx,
first_name, last_name, full_name, email_id, phone, mobile_no, status
) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open')
""", (cont_id, ts, ts, ADMIN, ADMIN,
first or cname, last or None, full or cname,
email or None, tel or None, cell or None))
# Contact → Customer link (Dynamic Link)
pgc.execute("""
INSERT INTO "tabDynamic Link" (
name, creation, modified, modified_by, owner, docstatus, idx,
link_doctype, link_name, link_title, parent, parentfield, parenttype
) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact')
""", (uid("DL-"), ts, ts, ADMIN, ADMIN, cust_id, cname, cont_id))
# Contact Email (child table)
if email:
pgc.execute("""
INSERT INTO "tabContact Email" (
name, creation, modified, modified_by, owner, docstatus, idx,
email_id, is_primary, parent, parentfield, parenttype
) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact')
""", (uid("CE-"), ts, ts, ADMIN, ADMIN, email, cont_id))
# Contact Phone (child table)
pidx = 1
if tel:
pgc.execute("""
INSERT INTO "tabContact Phone" (
name, creation, modified, modified_by, owner, docstatus, idx,
phone, is_primary_phone, is_primary_mobile_no,
parent, parentfield, parenttype
) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact')
""", (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, tel, cont_id))
pidx += 1
if cell:
pgc.execute("""
INSERT INTO "tabContact Phone" (
name, creation, modified, modified_by, owner, docstatus, idx,
phone, is_primary_phone, is_primary_mobile_no,
parent, parentfield, parenttype
) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact')
""", (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, cell, cont_id))
c_contact += 1
# --- Addresses ---
for j, d in enumerate(del_by.get(aid, [])):
addr1 = clean(d.get("address1"))
city = clean(d.get("city"))
if not addr1 and not city:
continue
addr_id = uid("ADDR-")
title = clean(d.get("name")) or cname
pgc.execute("""
INSERT INTO "tabAddress" (
name, creation, modified, modified_by, owner, docstatus, idx,
address_title, address_type, address_line1, city, state,
pincode, country, is_primary_address, is_shipping_address
) VALUES (%s, %s, %s, %s, %s, 0, 0,
%s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1)
""", (addr_id, ts, ts, ADMIN, ADMIN,
title, addr1 or "N/A", city or "N/A",
clean(d.get("state")) or "QC", clean(d.get("zip")),
1 if j == 0 else 0))
# Address → Customer link
pgc.execute("""
INSERT INTO "tabDynamic Link" (
name, creation, modified, modified_by, owner, docstatus, idx,
link_doctype, link_name, link_title, parent, parentfield, parenttype
) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address')
""", (uid("DL-"), ts, ts, ADMIN, ADMIN, j+1, cust_id, cname, addr_id))
c_addr += 1
except Exception as e:
c_err += 1
pg.rollback()
if c_err <= 20:
print(" ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100]), flush=True)
continue
# Commit every 200
if c_ok % 200 == 0:
pg.commit()
print(" [{}/{}] cust={} addr={} contact={} skip={} err={}".format(
i+1, len(accounts), c_ok, c_addr, c_contact, c_skip, c_err), flush=True)
pg.commit()
pg.close()
print("", flush=True)
print("=" * 60, flush=True)
print("Customers: {} created, {} skipped, {} errors".format(c_ok, c_skip, c_err), flush=True)
print("Contacts: {}".format(c_contact), flush=True)
print("Addresses: {}".format(c_addr), flush=True)
print("=" * 60, flush=True)
print("", flush=True)
print("Next: run 'bench --site erp.gigafibre.ca clear-cache'", flush=True)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink