gigafibre-fsm/scripts/migration/migrate_tickets.py

#!/usr/bin/env python3
"""
Phase 6: Import tickets → ERPNext Issue + Communication.
- Creates Issue Types from ticket_dept
- Imports open+pending tickets (1,395) as Issues
- Imports closed tickets (241K) as closed Issues
- Imports ticket_msg as Communication
- Maps parent/child (incident pattern)

Direct PostgreSQL. Detached.
Log: /tmp/migrate_tickets.log
"""
import pymysql
import psycopg2
import uuid
from datetime import datetime, timezone
from html import unescape

LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
          "database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
      "dbname": "_eb65bdc0c4b1b2d6"}

ADMIN = "Administrator"
COMPANY = "TARGO"

# Priority mapping: legacy (0=urgent,1=high,2=normal,3=low)
PRIORITY_MAP = {0: "Urgent", 1: "High", 2: "Medium", 3: "Low"}
STATUS_MAP = {"open": "Open", "pending": "On Hold", "closed": "Closed"}

def uid(prefix=""):
    return prefix + uuid.uuid4().hex[:10]

def now():
    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")

def ts_to_date(unix_ts):
    if not unix_ts or unix_ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
    except (ValueError, OSError):
        return None

def ts_to_dateonly(unix_ts):
    if not unix_ts or unix_ts <= 0:
        return None
    try:
        return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d")
    except (ValueError, OSError):
        return None

def clean(val):
    if not val:
        return ""
    return unescape(str(val)).strip()

def log(msg):
    print(msg, flush=True)

def main():
    ts = now()
    log("=== Phase 6: Tickets Migration ===")

    # 1. Read legacy data
    log("Reading legacy data...")
    mc = pymysql.connect(**LEGACY)
    cur = mc.cursor(pymysql.cursors.DictCursor)

    cur.execute("SELECT * FROM ticket_dept ORDER BY id")
    depts = cur.fetchall()

    cur.execute("SELECT id, username, first_name, last_name, email FROM staff ORDER BY id")
    staff_list = cur.fetchall()

    # ALL tickets (open, pending, closed) — only needed columns (avoid wizard/wizard_fibre blobs)
    cur.execute("""SELECT id, account_id, delivery_id, subject, status, priority,
                          dept_id, date_create, parent, open_by, assign_to, important
                   FROM ticket ORDER BY id""")
    tickets = cur.fetchall()

    # Messages for open/pending tickets + last message for closed (for context)
    cur.execute("""
        SELECT m.id, m.ticket_id, m.staff_id, m.msg, m.date_orig FROM ticket_msg m
        JOIN ticket t ON m.ticket_id = t.id
        WHERE t.status IN ('open', 'pending')
        ORDER BY m.ticket_id, m.id
    """)
    open_msgs = cur.fetchall()

    # For closed tickets: just count (we'll import messages in batches later)
    cur.execute("SELECT COUNT(*) as cnt FROM ticket_msg WHERE ticket_id IN (SELECT id FROM ticket WHERE status = 'closed')")
    closed_msg_count = cur.fetchone()["cnt"]

    mc.close()
    log("  {} depts, {} staff, {} tickets".format(len(depts), len(staff_list), len(tickets)))
    log("  {} messages for open/pending tickets".format(len(open_msgs)))
    log("  {} messages for closed tickets (deferred)".format(closed_msg_count))

    # 2. Connect ERPNext PG
    log("Connecting to ERPNext PostgreSQL...")
    pg = psycopg2.connect(**PG)
    pgc = pg.cursor()

    # Customer mapping
    pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0')
    cust_map = {r[0]: r[1] for r in pgc.fetchall()}

    # Staff → User mapping (by email)
    pgc.execute('SELECT name, email FROM "tabUser" WHERE enabled = 1')
    user_by_email = {r[1]: r[0] for r in pgc.fetchall() if r[1]}

    staff_to_user = {}
    for s in staff_list:
        email = s.get("email", "")
        if email and email in user_by_email:
            staff_to_user[s["id"]] = user_by_email[email]

    # Existing Issue Types
    pgc.execute('SELECT name FROM "tabIssue Type"')
    existing_types = set(r[0] for r in pgc.fetchall())

    # Existing Issues by legacy_ticket_id
    pgc.execute('SELECT legacy_ticket_id FROM "tabIssue" WHERE legacy_ticket_id IS NOT NULL AND legacy_ticket_id > 0')
    existing_issues = set(r[0] for r in pgc.fetchall())

    # Issue Priority
    pgc.execute('SELECT name FROM "tabIssue Priority"')
    existing_priorities = set(r[0] for r in pgc.fetchall())

    log("  {} customers mapped, {} staff→user mapped".format(len(cust_map), len(staff_to_user)))

    # 3. Create Issue Priorities
    for pname in ["Urgent", "High", "Medium", "Low"]:
        if pname not in existing_priorities:
            pgc.execute("""
                INSERT INTO "tabIssue Priority" (name, creation, modified, modified_by, owner, docstatus, idx)
                VALUES (%s, %s, %s, %s, %s, 0, 0)
            """, (pname, ts, ts, ADMIN, ADMIN))
    pg.commit()
    log("  Issue Priorities created")

    # 4. Create Issue Types from ticket_dept
    log("")
    log("--- Creating Issue Types ---")
    dept_map = {}  # dept_id → Issue Type name
    types_created = 0

    for d in depts:
        name = clean(d["name"])
        if not name:
            continue
        dept_map[d["id"]] = name
        if name in existing_types:
            continue
        pgc.execute("""
            INSERT INTO "tabIssue Type" (name, creation, modified, modified_by, owner, docstatus, idx)
            VALUES (%s, %s, %s, %s, %s, 0, 0)
        """, (name, ts, ts, ADMIN, ADMIN))
        existing_types.add(name)
        types_created += 1

    pg.commit()
    log("  {} Issue Types created".format(types_created))

    # 5. Import Tickets as Issues
    log("")
    log("--- Importing Tickets → Issues ---")

    # Build message lookup by ticket_id
    msgs_by_ticket = {}
    for m in open_msgs:
        msgs_by_ticket.setdefault(m["ticket_id"], []).append(m)

    # Track legacy_ticket_id → ERPNext issue name for parent mapping
    ticket_to_issue = {}
    i_ok = i_skip = i_err = 0
    comm_ok = 0

    for i, t in enumerate(tickets):
        tid = t["id"]
        if tid in existing_issues:
            i_skip += 1
            continue

        subject = clean(t.get("subject")) or "Ticket #{}".format(tid)
        status = STATUS_MAP.get(t.get("status", "open"), "Open")
        priority = PRIORITY_MAP.get(t.get("priority", 2), "Medium")
        dept_name = dept_map.get(t.get("dept_id"), None)
        cust_name = cust_map.get(t.get("account_id"))
        is_important = 1 if t.get("important") else 0

        opening_date = ts_to_dateonly(t.get("date_create"))
        opening_time = None
        if t.get("date_create") and t["date_create"] > 0:
            try:
                opening_time = datetime.fromtimestamp(int(t["date_create"]), tz=timezone.utc).strftime("%H:%M:%S")
            except (ValueError, OSError):
                pass

        # Is this an incident (parent)?
        is_incident = 0
        # Check if other tickets reference this one as parent
        # We'll do a second pass for this

        issue_name = uid("ISS-")

        try:
            # Savepoint so errors only rollback THIS ticket, not the whole batch
            pgc.execute("SAVEPOINT sp_ticket")
            pgc.execute("""
                INSERT INTO "tabIssue" (
                    name, creation, modified, modified_by, owner, docstatus, idx,
                    naming_series, subject, status, priority, issue_type,
                    customer, company, opening_date, opening_time,
                    legacy_ticket_id, is_incident, is_important,
                    parent_incident, service_location
                ) VALUES (
                    %s, %s, %s, %s, %s, 0, 0,
                    'ISS-.YYYY.-', %s, %s, %s, %s,
                    %s, %s, %s, %s,
                    %s, %s, %s, %s, %s
                )
            """, (issue_name, ts, ts, ADMIN, ADMIN,
                  subject[:255], status, priority, dept_name,
                  cust_name, COMPANY, opening_date, opening_time,
                  tid, 0, is_important, None, None))

            ticket_to_issue[tid] = issue_name
            i_ok += 1

            # Import messages for open/pending tickets
            for m in msgs_by_ticket.get(tid, []):
                msg_text = m.get("msg") or ""
                if not msg_text.strip():
                    continue
                sender = staff_to_user.get(m.get("staff_id"), ADMIN)
                msg_date = ts_to_date(m.get("date_orig"))

                comm_name = uid("COM-")
                try:
                    pgc.execute("""
                        INSERT INTO "tabComment" (
                            name, creation, modified, modified_by, owner, docstatus, idx,
                            comment_type, comment_by, content,
                            reference_doctype, reference_name
                        ) VALUES (
                            %s, %s, %s, %s, %s, 0, 0,
                            'Comment', %s, %s,
                            'Issue', %s
                        )
                    """, (comm_name, msg_date or ts, msg_date or ts, ADMIN, ADMIN,
                          sender, msg_text,
                          issue_name))
                    comm_ok += 1
                except Exception:
                    pgc.execute("ROLLBACK TO SAVEPOINT sp_ticket")
                    pgc.execute("SAVEPOINT sp_ticket")
                    # Skip message but keep the Issue

            pgc.execute("RELEASE SAVEPOINT sp_ticket")

        except Exception as e:
            i_err += 1
            pgc.execute("ROLLBACK TO SAVEPOINT sp_ticket")
            pgc.execute("RELEASE SAVEPOINT sp_ticket")
            if i_err <= 20:
                log("  ERR ticket#{} -> {}".format(tid, str(e)[:100]))
            continue

        if i_ok % 2000 == 0:
            pg.commit()
            log("  [{}/{}] issues={} comm={} skip={} err={}".format(
                i+1, len(tickets), i_ok, comm_ok, i_skip, i_err))

    pg.commit()

    # 6. Second pass: set parent_incident for sub-tickets
    log("")
    log("--- Setting parent/child relationships ---")
    parent_set = 0
    incident_set = 0

    for t in tickets:
        if t.get("parent") and t["parent"] > 0:
            child_issue = ticket_to_issue.get(t["id"])
            parent_issue = ticket_to_issue.get(t["parent"])
            if child_issue and parent_issue:
                pgc.execute("""
                    UPDATE "tabIssue" SET parent_incident = %s WHERE name = %s
                """, (parent_issue, child_issue))
                parent_set += 1

                # Mark parent as incident
                pgc.execute("""
                    UPDATE "tabIssue" SET is_incident = 1 WHERE name = %s AND is_incident = 0
                """, (parent_issue,))
                incident_set += 1

    # Update affected_clients count for incidents
    pgc.execute("""
        UPDATE "tabIssue" i SET affected_clients = (
            SELECT COUNT(*) FROM "tabIssue" child WHERE child.parent_incident = i.name
        ) WHERE i.is_incident = 1
    """)

    pg.commit()

    log("  {} parent links set, {} incidents identified".format(parent_set, incident_set))

    # 7. Update is_important on ALL tickets (existing + new)
    log("")
    log("--- Updating is_important flag ---")
    imp_map = {t["id"]: 1 for t in tickets if t.get("important")}
    imp_updated = 0
    if imp_map:
        imp_ids = list(imp_map.keys())
        pgc.execute("""
            UPDATE "tabIssue" SET is_important = 1
            WHERE legacy_ticket_id = ANY(%s) AND (is_important IS NULL OR is_important = 0)
        """, (imp_ids,))
        imp_updated = pgc.rowcount
        pg.commit()
    log("  {} tickets marked as important".format(imp_updated))

    pg.close()

    log("")
    log("=" * 60)
    log("Issue Types:    {} created".format(types_created))
    log("Issues:         {} created, {} skipped, {} errors".format(i_ok, i_skip, i_err))
    log("Communications: {} (open/pending tickets only)".format(comm_ok))
    log("Parent/child:   {} links, {} incidents".format(parent_set, incident_set))
    log("Closed msgs:    {} deferred (import separately if needed)".format(closed_msg_count))
    log("=" * 60)
    log("")
    log("Next: bench --site erp.gigafibre.ca clear-cache")

if __name__ == "__main__":
    main()