feat: complete data mirror — all customers + 115K invoices
- 8,636 terminated customers imported (disabled=1, terminate reason/company/notes preserved) - Total customers: 15,303 (100% of legacy) - 33,131 Subscription.party links fixed (CUST-xxx) - 115,721 Sales Invoices (24 months) + 658K line items - Custom field: Sales Invoice.legacy_invoice_id - All invoices as Draft (not submitted, not sent) Customer lifecycle preserved: Active → services, subscriptions, invoices Terminated → disabled=1, customer_details has departure reason/competitor Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
22377bb381
commit
4f74376412
274
scripts/migration/fix_and_invoices.py
Normal file
274
scripts/migration/fix_and_invoices.py
Normal file
|
|
@ -0,0 +1,274 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
1. Fix Subscription.party → Customer.name (CUST-xxx)
|
||||||
|
2. Import legacy invoices (last 24 months) as Sales Invoice
|
||||||
|
Direct PG. Detached.
|
||||||
|
"""
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
|
LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
|
||||||
|
"database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
|
||||||
|
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
|
||||||
|
"dbname": "_eb65bdc0c4b1b2d6"}
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
COMPANY = "TARGO"
|
||||||
|
|
||||||
|
def uid(p=""):
|
||||||
|
return p + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
|
||||||
|
def ts_to_date(ts):
|
||||||
|
if not ts or ts <= 0:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(int(ts), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
except:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def clean(v):
|
||||||
|
if not v: return ""
|
||||||
|
return unescape(str(v)).strip()
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print(msg, flush=True)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ts = now()
|
||||||
|
|
||||||
|
# =============================
|
||||||
|
# PART 1: Fix Subscription.party
|
||||||
|
# =============================
|
||||||
|
log("=== Part 1: Fix Subscription.party ===")
|
||||||
|
|
||||||
|
pg = psycopg2.connect(**PG)
|
||||||
|
pgc = pg.cursor()
|
||||||
|
|
||||||
|
# Fix party field: match customer_name → actual name
|
||||||
|
pgc.execute("""
|
||||||
|
UPDATE "tabSubscription" s
|
||||||
|
SET party = c.name
|
||||||
|
FROM "tabCustomer" c
|
||||||
|
WHERE s.party = c.customer_name
|
||||||
|
AND s.party_type = 'Customer'
|
||||||
|
AND s.party NOT LIKE 'CUST-%'
|
||||||
|
""")
|
||||||
|
fixed_party = pgc.rowcount
|
||||||
|
pg.commit()
|
||||||
|
log(" Fixed {} Subscription.party → CUST-xxx".format(fixed_party))
|
||||||
|
|
||||||
|
# Also fix any remaining by legacy_service_id mapping
|
||||||
|
pgc.execute("""
|
||||||
|
SELECT s.name as sub_name, s.legacy_service_id
|
||||||
|
FROM "tabSubscription" s
|
||||||
|
WHERE s.party NOT LIKE 'CUST-%' AND s.legacy_service_id > 0
|
||||||
|
""")
|
||||||
|
remaining = pgc.fetchall()
|
||||||
|
|
||||||
|
if remaining:
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
cur.execute("SELECT s.id, d.account_id FROM service s JOIN delivery d ON s.delivery_id = d.id WHERE s.status = 1")
|
||||||
|
svc_acct = {r["id"]: r["account_id"] for r in cur.fetchall()}
|
||||||
|
mc.close()
|
||||||
|
|
||||||
|
pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||||
|
cust_map = {r[0]: r[1] for r in pgc.fetchall()}
|
||||||
|
|
||||||
|
fixed2 = 0
|
||||||
|
for sub_name, svc_id in remaining:
|
||||||
|
acct_id = svc_acct.get(svc_id)
|
||||||
|
if acct_id:
|
||||||
|
cust_name = cust_map.get(acct_id)
|
||||||
|
if cust_name:
|
||||||
|
pgc.execute('UPDATE "tabSubscription" SET party = %s WHERE name = %s', (cust_name, sub_name))
|
||||||
|
fixed2 += 1
|
||||||
|
pg.commit()
|
||||||
|
log(" Fixed {} more via legacy_service_id".format(fixed2))
|
||||||
|
else:
|
||||||
|
log(" No remaining fixes needed")
|
||||||
|
# Still need legacy data for Part 2
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
|
||||||
|
# =============================
|
||||||
|
# PART 2: Import Invoices
|
||||||
|
# =============================
|
||||||
|
log("")
|
||||||
|
log("=== Part 2: Import Legacy Invoices (24 months) ===")
|
||||||
|
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
|
||||||
|
# Invoices from last 24 months
|
||||||
|
cutoff = int((datetime.now(timezone.utc).timestamp())) - (24 * 30 * 86400)
|
||||||
|
cur.execute("""
|
||||||
|
SELECT i.id, i.account_id, i.date_orig, i.total_amt, i.billed_amt,
|
||||||
|
i.billing_status, i.due_date, i.notes
|
||||||
|
FROM invoice i
|
||||||
|
WHERE i.billing_status = 1 AND i.date_orig >= %s
|
||||||
|
ORDER BY i.id
|
||||||
|
""", (cutoff,))
|
||||||
|
invoices = cur.fetchall()
|
||||||
|
log(" {} invoices to import".format(len(invoices)))
|
||||||
|
|
||||||
|
# Invoice items
|
||||||
|
inv_ids = [i["id"] for i in invoices]
|
||||||
|
items_by_inv = {}
|
||||||
|
if inv_ids:
|
||||||
|
# Batch query in chunks
|
||||||
|
chunk = 10000
|
||||||
|
for start in range(0, len(inv_ids), chunk):
|
||||||
|
batch = inv_ids[start:start+chunk]
|
||||||
|
placeholders = ",".join(["%s"] * len(batch))
|
||||||
|
cur.execute("""
|
||||||
|
SELECT invoice_id, sku, quantity, unitary_price, product_name, service_id
|
||||||
|
FROM invoice_item WHERE invoice_id IN ({})
|
||||||
|
""".format(placeholders), batch)
|
||||||
|
for r in cur.fetchall():
|
||||||
|
items_by_inv.setdefault(r["invoice_id"], []).append(r)
|
||||||
|
mc.close()
|
||||||
|
log(" {} invoice items loaded".format(sum(len(v) for v in items_by_inv.values())))
|
||||||
|
|
||||||
|
# Customer mapping
|
||||||
|
pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||||
|
cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()}
|
||||||
|
|
||||||
|
# Check existing invoices
|
||||||
|
pgc.execute('SELECT name FROM "tabSales Invoice" WHERE name LIKE %s', ('SINV-LEG-%',))
|
||||||
|
existing_inv = set(r[0] for r in pgc.fetchall())
|
||||||
|
|
||||||
|
# Item existence check
|
||||||
|
pgc.execute('SELECT item_code FROM "tabItem"')
|
||||||
|
valid_items = set(r[0] for r in pgc.fetchall())
|
||||||
|
|
||||||
|
# Get receivable + income accounts
|
||||||
|
pgc.execute("""SELECT name FROM "tabAccount" WHERE account_type = 'Receivable' AND company = 'TARGO' AND is_group = 0 LIMIT 1""")
|
||||||
|
receivable = pgc.fetchone()[0]
|
||||||
|
pgc.execute("""SELECT name FROM "tabAccount" WHERE root_type = 'Income' AND company = 'TARGO' AND is_group = 0 LIMIT 1""")
|
||||||
|
income_row = pgc.fetchone()
|
||||||
|
income_acct = income_row[0] if income_row else "Revenus autres - T"
|
||||||
|
|
||||||
|
inv_ok = inv_skip = inv_err = item_ok = 0
|
||||||
|
|
||||||
|
for i, inv in enumerate(invoices):
|
||||||
|
inv_name = "SINV-LEG-{}".format(inv["id"])
|
||||||
|
if inv_name in existing_inv:
|
||||||
|
inv_skip += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cust_data = cust_map.get(inv["account_id"])
|
||||||
|
if not cust_data:
|
||||||
|
inv_err += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cust_name, cust_display = cust_data
|
||||||
|
posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01"
|
||||||
|
due_date = ts_to_date(inv["due_date"]) or posting_date
|
||||||
|
total = float(inv["total_amt"] or 0)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Sales Invoice header
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
naming_series, title, customer, customer_name, company,
|
||||||
|
posting_date, due_date, currency, conversion_rate,
|
||||||
|
selling_price_list, price_list_currency,
|
||||||
|
base_grand_total, grand_total, base_net_total, net_total,
|
||||||
|
base_total, total,
|
||||||
|
outstanding_amount, base_rounded_total, rounded_total,
|
||||||
|
is_return, is_debit_note, disable_rounded_total,
|
||||||
|
debit_to, party_account_currency,
|
||||||
|
status, docstatus
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
'ACC-SINV-.YYYY.-', %s, %s, %s, %s,
|
||||||
|
%s, %s, 'CAD', 1,
|
||||||
|
'Standard Selling', 'CAD',
|
||||||
|
%s, %s, %s, %s,
|
||||||
|
%s, %s,
|
||||||
|
%s, %s, %s,
|
||||||
|
0, 0, 1,
|
||||||
|
%s, 'CAD',
|
||||||
|
'Draft', 0
|
||||||
|
)
|
||||||
|
""", (inv_name, ts, ts, ADMIN, ADMIN,
|
||||||
|
cust_display, cust_name, cust_display, COMPANY,
|
||||||
|
posting_date, due_date,
|
||||||
|
total, total, total, total,
|
||||||
|
total, total,
|
||||||
|
total, total, total,
|
||||||
|
receivable))
|
||||||
|
|
||||||
|
# Invoice items
|
||||||
|
line_items = items_by_inv.get(inv["id"], [])
|
||||||
|
for j, li in enumerate(line_items):
|
||||||
|
sku = clean(li.get("sku")) or "MISC"
|
||||||
|
qty = float(li.get("quantity") or 1)
|
||||||
|
rate = float(li.get("unitary_price") or 0)
|
||||||
|
amount = round(qty * rate, 2)
|
||||||
|
desc = clean(li.get("product_name")) or sku
|
||||||
|
|
||||||
|
# Use valid item or fallback
|
||||||
|
item_code = sku if sku in valid_items else None
|
||||||
|
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice Item" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
item_code, item_name, description, qty, rate, amount,
|
||||||
|
base_rate, base_amount, base_net_rate, base_net_amount,
|
||||||
|
net_rate, net_amount,
|
||||||
|
stock_uom, uom, conversion_factor,
|
||||||
|
income_account, cost_center,
|
||||||
|
parent, parentfield, parenttype
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, %s,
|
||||||
|
%s, %s, %s, %s, %s, %s,
|
||||||
|
%s, %s, %s, %s,
|
||||||
|
%s, %s,
|
||||||
|
'Nos', 'Nos', 1,
|
||||||
|
%s, 'Main - T',
|
||||||
|
%s, 'items', 'Sales Invoice'
|
||||||
|
)
|
||||||
|
""", (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1,
|
||||||
|
item_code, desc[:140], desc[:140], qty, rate, amount,
|
||||||
|
rate, amount, rate, amount,
|
||||||
|
rate, amount,
|
||||||
|
income_acct,
|
||||||
|
inv_name))
|
||||||
|
item_ok += 1
|
||||||
|
|
||||||
|
inv_ok += 1
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
inv_err += 1
|
||||||
|
pg.rollback()
|
||||||
|
if inv_err <= 10:
|
||||||
|
log(" ERR inv#{} -> {}".format(inv["id"], str(e)[:100]))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if inv_ok % 2000 == 0:
|
||||||
|
pg.commit()
|
||||||
|
log(" [{}/{}] inv={} items={} skip={} err={}".format(
|
||||||
|
i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err))
|
||||||
|
|
||||||
|
pg.commit()
|
||||||
|
pg.close()
|
||||||
|
|
||||||
|
log("")
|
||||||
|
log("=" * 60)
|
||||||
|
log("Subscriptions fixed: {}".format(fixed_party))
|
||||||
|
log("Invoices: {} created, {} skipped, {} errors".format(inv_ok, inv_skip, inv_err))
|
||||||
|
log("Invoice Items: {}".format(item_ok))
|
||||||
|
log("=" * 60)
|
||||||
|
log("bench --site erp.gigafibre.ca clear-cache")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
182
scripts/migration/import_invoices.py
Normal file
182
scripts/migration/import_invoices.py
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Import legacy invoices (24 months) as Sales Invoice drafts. Direct PG."""
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
|
LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
|
||||||
|
"database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
|
||||||
|
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
|
||||||
|
"dbname": "_eb65bdc0c4b1b2d6"}
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
COMPANY = "TARGO"
|
||||||
|
|
||||||
|
def uid(p=""):
|
||||||
|
return p + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
|
||||||
|
def ts_to_date(t):
|
||||||
|
if not t or t <= 0: return None
|
||||||
|
try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
except: return None
|
||||||
|
|
||||||
|
def clean(v):
|
||||||
|
if not v: return ""
|
||||||
|
return unescape(str(v)).strip()
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print(msg, flush=True)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ts = now()
|
||||||
|
log("=== Import Invoices (24 months) ===")
|
||||||
|
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
cutoff = int(datetime.now(timezone.utc).timestamp()) - (24 * 30 * 86400)
|
||||||
|
|
||||||
|
cur.execute("""SELECT * FROM invoice WHERE billing_status = 1 AND date_orig >= %s ORDER BY id""", (cutoff,))
|
||||||
|
invoices = cur.fetchall()
|
||||||
|
log(" {} invoices".format(len(invoices)))
|
||||||
|
|
||||||
|
inv_ids = [i["id"] for i in invoices]
|
||||||
|
items_by_inv = {}
|
||||||
|
chunk = 10000
|
||||||
|
for s in range(0, len(inv_ids), chunk):
|
||||||
|
batch = inv_ids[s:s+chunk]
|
||||||
|
cur.execute("SELECT * FROM invoice_item WHERE invoice_id IN ({})".format(",".join(["%s"]*len(batch))), batch)
|
||||||
|
for r in cur.fetchall():
|
||||||
|
items_by_inv.setdefault(r["invoice_id"], []).append(r)
|
||||||
|
mc.close()
|
||||||
|
log(" {} items loaded".format(sum(len(v) for v in items_by_inv.values())))
|
||||||
|
|
||||||
|
pg = psycopg2.connect(**PG)
|
||||||
|
pgc = pg.cursor()
|
||||||
|
|
||||||
|
pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||||
|
cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()}
|
||||||
|
|
||||||
|
pgc.execute('SELECT item_code FROM "tabItem"')
|
||||||
|
valid_items = set(r[0] for r in pgc.fetchall())
|
||||||
|
|
||||||
|
pgc.execute("""SELECT name FROM "tabAccount" WHERE account_type = 'Receivable' AND company = %s AND is_group = 0 LIMIT 1""", (COMPANY,))
|
||||||
|
receivable = pgc.fetchone()[0]
|
||||||
|
|
||||||
|
pgc.execute("""SELECT name FROM "tabAccount" WHERE root_type = 'Income' AND company = %s AND is_group = 0 LIMIT 1""", (COMPANY,))
|
||||||
|
income_acct = pgc.fetchone()[0]
|
||||||
|
|
||||||
|
pgc.execute('SELECT legacy_invoice_id FROM "tabSales Invoice" WHERE legacy_invoice_id > 0')
|
||||||
|
existing = set(r[0] for r in pgc.fetchall())
|
||||||
|
log(" {} already exist".format(len(existing)))
|
||||||
|
|
||||||
|
inv_ok = inv_skip = inv_err = item_ok = 0
|
||||||
|
|
||||||
|
for i, inv in enumerate(invoices):
|
||||||
|
if inv["id"] in existing:
|
||||||
|
inv_skip += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cust_data = cust_map.get(inv["account_id"])
|
||||||
|
if not cust_data:
|
||||||
|
inv_err += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cust_name, cust_display = cust_data
|
||||||
|
posting_date = ts_to_date(inv["date_orig"]) or "2025-01-01"
|
||||||
|
due_date = ts_to_date(inv["due_date"]) or posting_date
|
||||||
|
total = round(float(inv["total_amt"] or 0), 2)
|
||||||
|
sinv_name = uid("SINV-")
|
||||||
|
|
||||||
|
try:
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
naming_series, customer, customer_name, company,
|
||||||
|
posting_date, due_date, currency, conversion_rate,
|
||||||
|
selling_price_list, price_list_currency,
|
||||||
|
base_grand_total, grand_total, base_net_total, net_total,
|
||||||
|
base_total, total,
|
||||||
|
outstanding_amount, base_rounded_total, rounded_total,
|
||||||
|
is_return, is_debit_note, disable_rounded_total,
|
||||||
|
debit_to, party_account_currency,
|
||||||
|
status, legacy_invoice_id
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
'ACC-SINV-.YYYY.-', %s, %s, %s,
|
||||||
|
%s, %s, 'CAD', 1,
|
||||||
|
'Standard Selling', 'CAD',
|
||||||
|
%s, %s, %s, %s,
|
||||||
|
%s, %s,
|
||||||
|
%s, %s, %s,
|
||||||
|
0, 0, 1,
|
||||||
|
%s, 'CAD',
|
||||||
|
'Draft', %s
|
||||||
|
)
|
||||||
|
""", (sinv_name, ts, ts, ADMIN, ADMIN,
|
||||||
|
cust_name, cust_display, COMPANY,
|
||||||
|
posting_date, due_date,
|
||||||
|
total, total, total, total,
|
||||||
|
total, total,
|
||||||
|
total, total, total,
|
||||||
|
receivable, inv["id"]))
|
||||||
|
|
||||||
|
for j, li in enumerate(items_by_inv.get(inv["id"], [])):
|
||||||
|
sku = clean(li.get("sku")) or "MISC"
|
||||||
|
qty = float(li.get("quantity") or 1)
|
||||||
|
rate = float(li.get("unitary_price") or 0)
|
||||||
|
amount = round(qty * rate, 2)
|
||||||
|
desc = clean(li.get("product_name")) or sku
|
||||||
|
item_code = sku if sku in valid_items else None
|
||||||
|
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice Item" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
item_code, item_name, description, qty, rate, amount,
|
||||||
|
base_rate, base_amount, base_net_rate, base_net_amount,
|
||||||
|
net_rate, net_amount,
|
||||||
|
stock_uom, uom, conversion_factor,
|
||||||
|
income_account, cost_center,
|
||||||
|
parent, parentfield, parenttype
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, %s,
|
||||||
|
%s, %s, %s, %s, %s, %s,
|
||||||
|
%s, %s, %s, %s,
|
||||||
|
%s, %s,
|
||||||
|
'Nos', 'Nos', 1,
|
||||||
|
%s, 'Main - T',
|
||||||
|
%s, 'items', 'Sales Invoice'
|
||||||
|
)
|
||||||
|
""", (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1,
|
||||||
|
item_code, desc[:140], desc[:140], qty, rate, amount,
|
||||||
|
rate, amount, rate, amount,
|
||||||
|
rate, amount,
|
||||||
|
income_acct, sinv_name))
|
||||||
|
item_ok += 1
|
||||||
|
|
||||||
|
inv_ok += 1
|
||||||
|
except Exception as e:
|
||||||
|
inv_err += 1
|
||||||
|
pg.rollback()
|
||||||
|
if inv_err <= 10:
|
||||||
|
log(" ERR inv#{} -> {}".format(inv["id"], str(e)[:100]))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if inv_ok % 2000 == 0:
|
||||||
|
pg.commit()
|
||||||
|
log(" [{}/{}] inv={} items={} skip={} err={}".format(i+1, len(invoices), inv_ok, item_ok, inv_skip, inv_err))
|
||||||
|
|
||||||
|
pg.commit()
|
||||||
|
pg.close()
|
||||||
|
log("")
|
||||||
|
log("=" * 60)
|
||||||
|
log("Invoices: {} created, {} skipped, {} errors".format(inv_ok, inv_skip, inv_err))
|
||||||
|
log("Items: {}".format(item_ok))
|
||||||
|
log("=" * 60)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
174
scripts/migration/import_terminated.py
Normal file
174
scripts/migration/import_terminated.py
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Import terminated customers (status=4) with their terminate reason as Lead intelligence."""
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
|
LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
|
||||||
|
"database": "gestionclient", "connect_timeout": 30, "read_timeout": 300}
|
||||||
|
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
|
||||||
|
"dbname": "_eb65bdc0c4b1b2d6"}
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual",
|
||||||
|
7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"}
|
||||||
|
|
||||||
|
def uid(p=""):
|
||||||
|
return p + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
|
||||||
|
def clean(v):
|
||||||
|
if not v: return ""
|
||||||
|
return unescape(str(v)).strip()
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print(msg, flush=True)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ts = now()
|
||||||
|
log("=== Import Terminated Customers (status 3,4,5) ===")
|
||||||
|
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
cur.execute("SELECT * FROM account WHERE status IN (3,4,5) ORDER BY id")
|
||||||
|
accounts = cur.fetchall()
|
||||||
|
cur.execute("SELECT * FROM delivery WHERE account_id IN (SELECT id FROM account WHERE status IN (3,4,5)) ORDER BY account_id")
|
||||||
|
deliveries = cur.fetchall()
|
||||||
|
mc.close()
|
||||||
|
log(" {} terminated accounts, {} deliveries".format(len(accounts), len(deliveries)))
|
||||||
|
|
||||||
|
del_by = {}
|
||||||
|
for d in deliveries:
|
||||||
|
del_by.setdefault(d["account_id"], []).append(d)
|
||||||
|
|
||||||
|
pg = psycopg2.connect(**PG)
|
||||||
|
pgc = pg.cursor()
|
||||||
|
|
||||||
|
pgc.execute('SELECT legacy_account_id FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||||
|
existing = set(r[0] for r in pgc.fetchall())
|
||||||
|
|
||||||
|
c_ok = c_addr = c_skip = c_err = 0
|
||||||
|
|
||||||
|
for i, a in enumerate(accounts):
|
||||||
|
aid = a["id"]
|
||||||
|
if aid in existing:
|
||||||
|
c_skip += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
first = clean(a["first_name"])
|
||||||
|
last = clean(a["last_name"])
|
||||||
|
company = clean(a["company"])
|
||||||
|
if company:
|
||||||
|
ctype, cname = "Company", company
|
||||||
|
else:
|
||||||
|
ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid)
|
||||||
|
|
||||||
|
cust_id = uid("CUST-")
|
||||||
|
group = GROUP_MAP.get(a["group_id"], "Individual")
|
||||||
|
|
||||||
|
# Build customer_details with terminate info
|
||||||
|
details = ""
|
||||||
|
if a.get("terminate_reason"):
|
||||||
|
details += "Raison départ: {}\n".format(clean(a["terminate_reason"]))
|
||||||
|
if a.get("terminate_cie"):
|
||||||
|
details += "Parti chez: {}\n".format(clean(a["terminate_cie"]))
|
||||||
|
if a.get("terminate_note"):
|
||||||
|
details += "Notes: {}\n".format(clean(a["terminate_note"])[:500])
|
||||||
|
if a.get("terminate_date"):
|
||||||
|
details += "Date: {}\n".format(clean(a["terminate_date"]))
|
||||||
|
|
||||||
|
try:
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabCustomer" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
naming_series, customer_name, customer_type, customer_group,
|
||||||
|
territory, default_currency, language, disabled,
|
||||||
|
legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id,
|
||||||
|
customer_pos_id, customer_details
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
'CUST-.YYYY.-', %s, %s, %s,
|
||||||
|
'Canada', 'CAD', 'fr', 1,
|
||||||
|
%s, %s, 0, %s,
|
||||||
|
%s, %s
|
||||||
|
)
|
||||||
|
""", (cust_id, ts, ts, ADMIN, ADMIN,
|
||||||
|
cname, ctype, group,
|
||||||
|
aid, clean(a.get("customer_id")),
|
||||||
|
clean(a.get("stripe_id")) or None,
|
||||||
|
clean(a.get("customer_id")),
|
||||||
|
details or None))
|
||||||
|
c_ok += 1
|
||||||
|
|
||||||
|
# Contact
|
||||||
|
email = clean(a.get("email"))
|
||||||
|
if first or email:
|
||||||
|
cont_id = uid("CONT-")
|
||||||
|
full = "{} {}".format(first, last).strip()
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabContact" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
first_name, last_name, full_name, email_id, phone, mobile_no, status
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open')
|
||||||
|
""", (cont_id, ts, ts, ADMIN, ADMIN,
|
||||||
|
first or cname, last or None, full or cname,
|
||||||
|
email or None, clean(a.get("tel_home")) or None,
|
||||||
|
clean(a.get("cell")) or None))
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabDynamic Link" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
link_doctype, link_name, link_title, parent, parentfield, parenttype
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact')
|
||||||
|
""", (uid("DL-"), ts, ts, ADMIN, ADMIN, cust_id, cname, cont_id))
|
||||||
|
|
||||||
|
# Addresses
|
||||||
|
for j, d in enumerate(del_by.get(aid, [])):
|
||||||
|
addr1 = clean(d.get("address1"))
|
||||||
|
city = clean(d.get("city"))
|
||||||
|
if not addr1 and not city: continue
|
||||||
|
addr_id = uid("ADDR-")
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabAddress" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
address_title, address_type, address_line1, city, state,
|
||||||
|
pincode, country, is_primary_address, is_shipping_address
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
%s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1)
|
||||||
|
""", (addr_id, ts, ts, ADMIN, ADMIN,
|
||||||
|
clean(d.get("name")) or cname, addr1 or "N/A", city or "N/A",
|
||||||
|
clean(d.get("state")) or "QC", clean(d.get("zip")),
|
||||||
|
1 if j == 0 else 0))
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabDynamic Link" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
link_doctype, link_name, link_title, parent, parentfield, parenttype
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address')
|
||||||
|
""", (uid("DL-"), ts, ts, ADMIN, ADMIN, j+1, cust_id, cname, addr_id))
|
||||||
|
c_addr += 1
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
c_err += 1
|
||||||
|
pg.rollback()
|
||||||
|
if c_err <= 10:
|
||||||
|
log(" ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:80]))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if c_ok % 500 == 0:
|
||||||
|
pg.commit()
|
||||||
|
log(" [{}/{}] created={} addr={} skip={} err={}".format(i+1, len(accounts), c_ok, c_addr, c_skip, c_err))
|
||||||
|
|
||||||
|
pg.commit()
|
||||||
|
pg.close()
|
||||||
|
log("")
|
||||||
|
log("=" * 60)
|
||||||
|
log("Terminated Customers: {} created, {} skipped, {} errors".format(c_ok, c_skip, c_err))
|
||||||
|
log("Addresses: {}".format(c_addr))
|
||||||
|
log("All terminated customers have disabled=1 + terminate details in customer_details")
|
||||||
|
log("=" * 60)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loading…
Reference in New Issue
Block a user