From 7a15bfd6008e36c203e3cbf762acdb7f84509142 Mon Sep 17 00:00:00 2001 From: louispaulb Date: Sat, 28 Mar 2026 15:06:58 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=206=20=E2=80=94=20242K=20tickets?= =?UTF-8?q?=20migrated=20as=20Issues=20with=20parent/child?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 38 Issue Types from ticket_dept - 242,605 Issues created (open + closed) - 25,805 parent/child links (incident pattern) - Custom fields: parent_incident, is_incident, affected_clients, impact_zone, service_location, legacy_ticket_id - Communications deferred (778K closed ticket messages — import separately) - 0 staff→user mapped (ERPNext users need to be created/linked) Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/migration/migrate_tickets.py | 314 +++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 scripts/migration/migrate_tickets.py diff --git a/scripts/migration/migrate_tickets.py b/scripts/migration/migrate_tickets.py new file mode 100644 index 0000000..46222d8 --- /dev/null +++ b/scripts/migration/migrate_tickets.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +""" +Phase 6: Import tickets → ERPNext Issue + Communication. +- Creates Issue Types from ticket_dept +- Imports open+pending tickets (1,395) as Issues +- Imports closed tickets (241K) as closed Issues +- Imports ticket_msg as Communication +- Maps parent/child (incident pattern) + +Direct PostgreSQL. Detached. +Log: /tmp/migrate_tickets.log +""" +import pymysql +import psycopg2 +import uuid +from datetime import datetime, timezone +from html import unescape + +LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj", + "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600} +PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123", + "dbname": "_eb65bdc0c4b1b2d6"} + +ADMIN = "Administrator" +COMPANY = "TARGO" + +# Priority mapping: legacy (0=urgent,1=high,2=normal,3=low) +PRIORITY_MAP = {0: "Urgent", 1: "High", 2: "Medium", 3: "Low"} +STATUS_MAP = {"open": "Open", "pending": "On Hold", "closed": "Closed"} + +def uid(prefix=""): + return prefix + uuid.uuid4().hex[:10] + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + +def ts_to_date(unix_ts): + if not unix_ts or unix_ts <= 0: + return None + try: + return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + except (ValueError, OSError): + return None + +def ts_to_dateonly(unix_ts): + if not unix_ts or unix_ts <= 0: + return None + try: + return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d") + except (ValueError, OSError): + return None + +def clean(val): + if not val: + return "" + return unescape(str(val)).strip() + +def log(msg): + print(msg, flush=True) + +def main(): + ts = now() + log("=== Phase 6: Tickets Migration ===") + + # 1. Read legacy data + log("Reading legacy data...") + mc = pymysql.connect(**LEGACY) + cur = mc.cursor(pymysql.cursors.DictCursor) + + cur.execute("SELECT * FROM ticket_dept ORDER BY id") + depts = cur.fetchall() + + cur.execute("SELECT id, username, first_name, last_name, email FROM staff ORDER BY id") + staff_list = cur.fetchall() + + # ALL tickets (open, pending, closed) + cur.execute("""SELECT * FROM ticket ORDER BY id""") + tickets = cur.fetchall() + + # Messages for open/pending tickets + last message for closed (for context) + cur.execute(""" + SELECT m.* FROM ticket_msg m + JOIN ticket t ON m.ticket_id = t.id + WHERE t.status IN ('open', 'pending') + ORDER BY m.ticket_id, m.id + """) + open_msgs = cur.fetchall() + + # For closed tickets: just count (we'll import messages in batches later) + cur.execute("SELECT COUNT(*) as cnt FROM ticket_msg WHERE ticket_id IN (SELECT id FROM ticket WHERE status = 'closed')") + closed_msg_count = cur.fetchone()["cnt"] + + mc.close() + log(" {} depts, {} staff, {} tickets".format(len(depts), len(staff_list), len(tickets))) + log(" {} messages for open/pending tickets".format(len(open_msgs))) + log(" {} messages for closed tickets (deferred)".format(closed_msg_count)) + + # 2. Connect ERPNext PG + log("Connecting to ERPNext PostgreSQL...") + pg = psycopg2.connect(**PG) + pgc = pg.cursor() + + # Customer mapping + pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0') + cust_map = {r[0]: r[1] for r in pgc.fetchall()} + + # Staff → User mapping (by email) + pgc.execute('SELECT name, email FROM "tabUser" WHERE enabled = 1') + user_by_email = {r[1]: r[0] for r in pgc.fetchall() if r[1]} + + staff_to_user = {} + for s in staff_list: + email = s.get("email", "") + if email and email in user_by_email: + staff_to_user[s["id"]] = user_by_email[email] + + # Existing Issue Types + pgc.execute('SELECT name FROM "tabIssue Type"') + existing_types = set(r[0] for r in pgc.fetchall()) + + # Existing Issues by legacy_ticket_id + pgc.execute('SELECT legacy_ticket_id FROM "tabIssue" WHERE legacy_ticket_id IS NOT NULL AND legacy_ticket_id > 0') + existing_issues = set(r[0] for r in pgc.fetchall()) + + # Issue Priority + pgc.execute('SELECT name FROM "tabIssue Priority"') + existing_priorities = set(r[0] for r in pgc.fetchall()) + + log(" {} customers mapped, {} staff→user mapped".format(len(cust_map), len(staff_to_user))) + + # 3. Create Issue Priorities + for pname in ["Urgent", "High", "Medium", "Low"]: + if pname not in existing_priorities: + pgc.execute(""" + INSERT INTO "tabIssue Priority" (name, creation, modified, modified_by, owner, docstatus, idx) + VALUES (%s, %s, %s, %s, %s, 0, 0) + """, (pname, ts, ts, ADMIN, ADMIN)) + pg.commit() + log(" Issue Priorities created") + + # 4. Create Issue Types from ticket_dept + log("") + log("--- Creating Issue Types ---") + dept_map = {} # dept_id → Issue Type name + types_created = 0 + + for d in depts: + name = clean(d["name"]) + if not name: + continue + dept_map[d["id"]] = name + if name in existing_types: + continue + pgc.execute(""" + INSERT INTO "tabIssue Type" (name, creation, modified, modified_by, owner, docstatus, idx) + VALUES (%s, %s, %s, %s, %s, 0, 0) + """, (name, ts, ts, ADMIN, ADMIN)) + existing_types.add(name) + types_created += 1 + + pg.commit() + log(" {} Issue Types created".format(types_created)) + + # 5. Import Tickets as Issues + log("") + log("--- Importing Tickets → Issues ---") + + # Build message lookup by ticket_id + msgs_by_ticket = {} + for m in open_msgs: + msgs_by_ticket.setdefault(m["ticket_id"], []).append(m) + + # Track legacy_ticket_id → ERPNext issue name for parent mapping + ticket_to_issue = {} + i_ok = i_skip = i_err = 0 + comm_ok = 0 + + for i, t in enumerate(tickets): + tid = t["id"] + if tid in existing_issues: + i_skip += 1 + continue + + subject = clean(t.get("subject")) or "Ticket #{}".format(tid) + status = STATUS_MAP.get(t.get("status", "open"), "Open") + priority = PRIORITY_MAP.get(t.get("priority", 2), "Medium") + dept_name = dept_map.get(t.get("dept_id"), None) + cust_name = cust_map.get(t.get("account_id")) + + opening_date = ts_to_dateonly(t.get("date_create")) + opening_time = None + if t.get("date_create") and t["date_create"] > 0: + try: + opening_time = datetime.fromtimestamp(int(t["date_create"]), tz=timezone.utc).strftime("%H:%M:%S") + except (ValueError, OSError): + pass + + # Is this an incident (parent)? + is_incident = 0 + # Check if other tickets reference this one as parent + # We'll do a second pass for this + + issue_name = uid("ISS-") + + try: + pgc.execute(""" + INSERT INTO "tabIssue" ( + name, creation, modified, modified_by, owner, docstatus, idx, + naming_series, subject, status, priority, issue_type, + customer, company, opening_date, opening_time, + legacy_ticket_id, is_incident, + parent_incident, service_location + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + 'ISS-.YYYY.-', %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, %s + ) + """, (issue_name, ts, ts, ADMIN, ADMIN, + subject[:255], status, priority, dept_name, + cust_name, COMPANY, opening_date, opening_time, + tid, 0, None, None)) + + ticket_to_issue[tid] = issue_name + i_ok += 1 + + # Import messages for open/pending tickets + for m in msgs_by_ticket.get(tid, []): + msg_text = m.get("msg") or "" + if not msg_text.strip(): + continue + sender = staff_to_user.get(m.get("staff_id"), ADMIN) + msg_date = ts_to_date(m.get("date_orig")) + + comm_name = uid("COM-") + pgc.execute(""" + INSERT INTO "tabCommunication" ( + name, creation, modified, modified_by, owner, docstatus, idx, + subject, content, communication_type, comment_type, + reference_doctype, reference_name, + sender, communication_date, sent_or_received + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + %s, %s, 'Communication', 'Comment', + 'Issue', %s, + %s, %s, 'Sent' + ) + """, (comm_name, ts, ts, ADMIN, ADMIN, + subject[:255], msg_text, + issue_name, + sender, msg_date or ts)) + comm_ok += 1 + + except Exception as e: + i_err += 1 + pg.rollback() + if i_err <= 20: + log(" ERR ticket#{} -> {}".format(tid, str(e)[:100])) + continue + + if i_ok % 2000 == 0: + pg.commit() + log(" [{}/{}] issues={} comm={} skip={} err={}".format( + i+1, len(tickets), i_ok, comm_ok, i_skip, i_err)) + + pg.commit() + + # 6. Second pass: set parent_incident for sub-tickets + log("") + log("--- Setting parent/child relationships ---") + parent_set = 0 + incident_set = 0 + + for t in tickets: + if t.get("parent") and t["parent"] > 0: + child_issue = ticket_to_issue.get(t["id"]) + parent_issue = ticket_to_issue.get(t["parent"]) + if child_issue and parent_issue: + pgc.execute(""" + UPDATE "tabIssue" SET parent_incident = %s WHERE name = %s + """, (parent_issue, child_issue)) + parent_set += 1 + + # Mark parent as incident + pgc.execute(""" + UPDATE "tabIssue" SET is_incident = 1 WHERE name = %s AND is_incident = 0 + """, (parent_issue,)) + incident_set += 1 + + # Update affected_clients count for incidents + pgc.execute(""" + UPDATE "tabIssue" i SET affected_clients = ( + SELECT COUNT(*) FROM "tabIssue" child WHERE child.parent_incident = i.name + ) WHERE i.is_incident = 1 + """) + + pg.commit() + pg.close() + + log(" {} parent links set, {} incidents identified".format(parent_set, incident_set)) + + log("") + log("=" * 60) + log("Issue Types: {} created".format(types_created)) + log("Issues: {} created, {} skipped, {} errors".format(i_ok, i_skip, i_err)) + log("Communications: {} (open/pending tickets only)".format(comm_ok)) + log("Parent/child: {} links, {} incidents".format(parent_set, incident_set)) + log("Closed msgs: {} deferred (import separately if needed)".format(closed_msg_count)) + log("=" * 60) + log("") + log("Next: bench --site erp.gigafibre.ca clear-cache") + +if __name__ == "__main__": + main()