gigafibre-fsm/scripts/migration/import_soumissions.py

"""
Import soumissions (quotes) from legacy as ERPNext Quotation.
Deserializes PHP-serialized materiel/mensuel arrays into Quotation Items.

Run inside erpnext-backend-1:
  /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/import_soumissions.py
"""
import frappe
import pymysql
import os, sys, time, re
from html import unescape

sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)
os.chdir("/home/frappe/frappe-bench/sites")
frappe.init(site="erp.gigafibre.ca", sites_path=".")
frappe.connect()
frappe.local.flags.ignore_permissions = True
print("Connected:", frappe.local.site)

legacy = pymysql.connect(
    host="legacy-db", user="facturation", password="VD67owoj",
    database="gestionclient", cursorclass=pymysql.cursors.DictCursor
)

cust_map = {}
for r in frappe.db.sql('SELECT name, legacy_account_id FROM "tabCustomer" WHERE legacy_account_id > 0', as_dict=True):
    cust_map[int(r['legacy_account_id'])] = r['name']
print(f"  {len(cust_map)} customers mapped")

# Ensure SVC item exists (catch-all for unmapped SKUs)
if not frappe.db.exists("Item", "SVC"):
    frappe.get_doc({"doctype": "Item", "item_code": "SVC", "item_name": "Service",
                    "item_group": "All Item Groups", "stock_uom": "Nos"}).insert()
    frappe.db.commit()
    print("  Created catch-all item SVC")


def parse_php_serialized(data):
    """Simple parser for PHP serialized arrays of items.
    Returns list of dicts with keys: sku, desc, amt, qte, tot"""
    if not data or data in ('a:0:{}', 'N;'):
        return []
    items = []
    # Extract items using regex — handles nested a:N:{...} structures
    # Pattern: s:3:"sku";s:N:"VALUE";...s:4:"desc";s:N:"VALUE";...s:3:"amt";s:N:"VALUE";
    item_pattern = re.compile(
        r's:3:"sku";s:\d+:"([^"]*)";'
        r's:4:"desc";s:\d+:"([^"]*)";'
        r's:3:"amt";s:\d+:"([^"]*)";'
        r's:3:"qte";s:\d+:"([^"]*)";'
        r's:3:"tot";s:\d+:"([^"]*)";'
    )
    for m in item_pattern.finditer(data):
        sku, desc, amt, qte, tot = m.groups()
        desc = unescape(desc).strip()
        try:
            amount = float(tot) if tot else (float(amt) * float(qte) if amt and qte else 0)
            qty = float(qte) if qte else 1
            rate = float(amt) if amt else (amount / qty if qty else 0)
        except ValueError:
            amount = 0
            qty = 1
            rate = 0
        if desc or amount:  # skip empty rows
            items.append({"sku": sku, "desc": desc, "rate": rate, "qty": qty, "amount": amount})
    return items


def parse_date(date_str):
    """Parse dd-mm-yyyy to yyyy-mm-dd"""
    if not date_str:
        return None
    try:
        parts = date_str.strip().split('-')
        if len(parts) == 3:
            return f"{parts[2]}-{parts[1]}-{parts[0]}"
    except Exception:
        pass
    return None


# Load soumissions
print("\n=== Loading legacy soumissions ===")
with legacy.cursor() as cur:
    cur.execute("SELECT * FROM soumission ORDER BY id")
    rows = cur.fetchall()
legacy.close()
print(f"  {len(rows)} soumissions to import")

# Add custom fields BEFORE cleanup check (column must exist first)
for fdef in [
    {"dt": "Quotation", "fieldname": "custom_legacy_soumission_id", "fieldtype": "Int",
     "label": "Legacy Soumission ID", "insert_after": "order_type"},
    {"dt": "Quotation", "fieldname": "custom_po_number", "fieldtype": "Data",
     "label": "PO Number", "insert_after": "custom_legacy_soumission_id"},
]:
    if not frappe.db.exists("Custom Field", {"dt": fdef["dt"], "fieldname": fdef["fieldname"]}):
        frappe.get_doc({"doctype": "Custom Field", **fdef}).insert(ignore_permissions=True)
        frappe.db.commit()
        print(f"  Added custom field {fdef['fieldname']}")

# Clear existing legacy quotations
existing = frappe.db.sql("SELECT COUNT(*) FROM \"tabQuotation\" WHERE custom_legacy_soumission_id > 0")[0][0]
if existing:
    # Delete items first, then quotations
    frappe.db.sql("""
        DELETE FROM "tabQuotation Item"
        WHERE parent IN (SELECT name FROM "tabQuotation" WHERE custom_legacy_soumission_id > 0)
    """)
    frappe.db.sql('DELETE FROM "tabQuotation" WHERE custom_legacy_soumission_id > 0')
    frappe.db.commit()
    print(f"  Cleared {existing} existing legacy Quotations")

T0 = time.time()
created = skipped = empty = 0

for r in rows:
    cust = cust_map.get(r['account_id'])
    if not cust:
        skipped += 1
        continue

    materiel = parse_php_serialized(r.get('materiel') or '')
    mensuel = parse_php_serialized(r.get('mensuel') or '')
    all_items = materiel + mensuel

    if not all_items:
        # Create with single placeholder item
        all_items = [{"sku": "SVC", "desc": (r['name'] or 'Soumission'), "rate": 0, "qty": 1, "amount": 0}]
        empty += 1

    posting_date = parse_date(r.get('date'))

    items = []
    for idx, item in enumerate(all_items):
        items.append({
            "item_code": "SVC",
            "item_name": item['desc'][:140] if item['desc'] else f"Item {idx+1}",
            "description": item['desc'] or '',
            "qty": item['qty'] or 1,
            "rate": item['rate'] or 0,
            "uom": "Nos",
        })

    doc = frappe.get_doc({
        "doctype": "Quotation",
        "quotation_to": "Customer",
        "party_name": cust,
        "transaction_date": posting_date or "2026-01-01",
        "valid_till": posting_date or "2026-01-01",
        "company": "TARGO",
        "currency": "CAD",
        "order_type": "Sales",
        "title": (r['name'] or f"Soumission {r['id']}")[:140],
        "terms": unescape(r.get('text') or ''),
        "custom_legacy_soumission_id": r['id'],
        "custom_po_number": r.get('po') or '',
        "items": items,
    })

    try:
        doc.insert(ignore_if_duplicate=True)
        created += 1
    except Exception as e:
        print(f"  ERR soumission {r['id']}: {str(e)[:80]}")
        skipped += 1

    if created % 100 == 0:
        frappe.db.commit()
        print(f"  [{created}/{len(rows)}] [{time.time()-T0:.0f}s]")

frappe.db.commit()

# ── Verify ──
print(f"\n=== Summary ===")
total = frappe.db.sql('SELECT COUNT(*) FROM "tabQuotation" WHERE custom_legacy_soumission_id > 0')[0][0]
print(f"  Created: {created} Quotations")
print(f"  Empty items (placeholder): {empty}")
print(f"  Skipped: {skipped}")
print(f"  Total: {total}")
print(f"  Time: {time.time()-T0:.0f}s")

frappe.destroy()
print("Done!")