feat: Phase 6 — 242K tickets migrated as Issues with parent/child
- 38 Issue Types from ticket_dept - 242,605 Issues created (open + closed) - 25,805 parent/child links (incident pattern) - Custom fields: parent_incident, is_incident, affected_clients, impact_zone, service_location, legacy_ticket_id - Communications deferred (778K closed ticket messages — import separately) - 0 staff→user mapped (ERPNext users need to be created/linked) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
louispaulb
parent
571f89976d
commit
7a15bfd600
314
scripts/migration/migrate_tickets.py
Normal file
314
scripts/migration/migrate_tickets.py
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 6: Import tickets → ERPNext Issue + Communication.
|
||||
- Creates Issue Types from ticket_dept
|
||||
- Imports open+pending tickets (1,395) as Issues
|
||||
- Imports closed tickets (241K) as closed Issues
|
||||
- Imports ticket_msg as Communication
|
||||
- Maps parent/child (incident pattern)
|
||||
|
||||
Direct PostgreSQL. Detached.
|
||||
Log: /tmp/migrate_tickets.log
|
||||
"""
|
||||
import pymysql
|
||||
import psycopg2
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj",
|
||||
"database": "gestionclient", "connect_timeout": 30, "read_timeout": 600}
|
||||
PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123",
|
||||
"dbname": "_eb65bdc0c4b1b2d6"}
|
||||
|
||||
ADMIN = "Administrator"
|
||||
COMPANY = "TARGO"
|
||||
|
||||
# Priority mapping: legacy (0=urgent,1=high,2=normal,3=low)
|
||||
PRIORITY_MAP = {0: "Urgent", 1: "High", 2: "Medium", 3: "Low"}
|
||||
STATUS_MAP = {"open": "Open", "pending": "On Hold", "closed": "Closed"}
|
||||
|
||||
def uid(prefix=""):
|
||||
return prefix + uuid.uuid4().hex[:10]
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
|
||||
def ts_to_date(unix_ts):
|
||||
if not unix_ts or unix_ts <= 0:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
def ts_to_dateonly(unix_ts):
|
||||
if not unix_ts or unix_ts <= 0:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(int(unix_ts), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
def clean(val):
|
||||
if not val:
|
||||
return ""
|
||||
return unescape(str(val)).strip()
|
||||
|
||||
def log(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
def main():
|
||||
ts = now()
|
||||
log("=== Phase 6: Tickets Migration ===")
|
||||
|
||||
# 1. Read legacy data
|
||||
log("Reading legacy data...")
|
||||
mc = pymysql.connect(**LEGACY)
|
||||
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||
|
||||
cur.execute("SELECT * FROM ticket_dept ORDER BY id")
|
||||
depts = cur.fetchall()
|
||||
|
||||
cur.execute("SELECT id, username, first_name, last_name, email FROM staff ORDER BY id")
|
||||
staff_list = cur.fetchall()
|
||||
|
||||
# ALL tickets (open, pending, closed)
|
||||
cur.execute("""SELECT * FROM ticket ORDER BY id""")
|
||||
tickets = cur.fetchall()
|
||||
|
||||
# Messages for open/pending tickets + last message for closed (for context)
|
||||
cur.execute("""
|
||||
SELECT m.* FROM ticket_msg m
|
||||
JOIN ticket t ON m.ticket_id = t.id
|
||||
WHERE t.status IN ('open', 'pending')
|
||||
ORDER BY m.ticket_id, m.id
|
||||
""")
|
||||
open_msgs = cur.fetchall()
|
||||
|
||||
# For closed tickets: just count (we'll import messages in batches later)
|
||||
cur.execute("SELECT COUNT(*) as cnt FROM ticket_msg WHERE ticket_id IN (SELECT id FROM ticket WHERE status = 'closed')")
|
||||
closed_msg_count = cur.fetchone()["cnt"]
|
||||
|
||||
mc.close()
|
||||
log(" {} depts, {} staff, {} tickets".format(len(depts), len(staff_list), len(tickets)))
|
||||
log(" {} messages for open/pending tickets".format(len(open_msgs)))
|
||||
log(" {} messages for closed tickets (deferred)".format(closed_msg_count))
|
||||
|
||||
# 2. Connect ERPNext PG
|
||||
log("Connecting to ERPNext PostgreSQL...")
|
||||
pg = psycopg2.connect(**PG)
|
||||
pgc = pg.cursor()
|
||||
|
||||
# Customer mapping
|
||||
pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||
cust_map = {r[0]: r[1] for r in pgc.fetchall()}
|
||||
|
||||
# Staff → User mapping (by email)
|
||||
pgc.execute('SELECT name, email FROM "tabUser" WHERE enabled = 1')
|
||||
user_by_email = {r[1]: r[0] for r in pgc.fetchall() if r[1]}
|
||||
|
||||
staff_to_user = {}
|
||||
for s in staff_list:
|
||||
email = s.get("email", "")
|
||||
if email and email in user_by_email:
|
||||
staff_to_user[s["id"]] = user_by_email[email]
|
||||
|
||||
# Existing Issue Types
|
||||
pgc.execute('SELECT name FROM "tabIssue Type"')
|
||||
existing_types = set(r[0] for r in pgc.fetchall())
|
||||
|
||||
# Existing Issues by legacy_ticket_id
|
||||
pgc.execute('SELECT legacy_ticket_id FROM "tabIssue" WHERE legacy_ticket_id IS NOT NULL AND legacy_ticket_id > 0')
|
||||
existing_issues = set(r[0] for r in pgc.fetchall())
|
||||
|
||||
# Issue Priority
|
||||
pgc.execute('SELECT name FROM "tabIssue Priority"')
|
||||
existing_priorities = set(r[0] for r in pgc.fetchall())
|
||||
|
||||
log(" {} customers mapped, {} staff→user mapped".format(len(cust_map), len(staff_to_user)))
|
||||
|
||||
# 3. Create Issue Priorities
|
||||
for pname in ["Urgent", "High", "Medium", "Low"]:
|
||||
if pname not in existing_priorities:
|
||||
pgc.execute("""
|
||||
INSERT INTO "tabIssue Priority" (name, creation, modified, modified_by, owner, docstatus, idx)
|
||||
VALUES (%s, %s, %s, %s, %s, 0, 0)
|
||||
""", (pname, ts, ts, ADMIN, ADMIN))
|
||||
pg.commit()
|
||||
log(" Issue Priorities created")
|
||||
|
||||
# 4. Create Issue Types from ticket_dept
|
||||
log("")
|
||||
log("--- Creating Issue Types ---")
|
||||
dept_map = {} # dept_id → Issue Type name
|
||||
types_created = 0
|
||||
|
||||
for d in depts:
|
||||
name = clean(d["name"])
|
||||
if not name:
|
||||
continue
|
||||
dept_map[d["id"]] = name
|
||||
if name in existing_types:
|
||||
continue
|
||||
pgc.execute("""
|
||||
INSERT INTO "tabIssue Type" (name, creation, modified, modified_by, owner, docstatus, idx)
|
||||
VALUES (%s, %s, %s, %s, %s, 0, 0)
|
||||
""", (name, ts, ts, ADMIN, ADMIN))
|
||||
existing_types.add(name)
|
||||
types_created += 1
|
||||
|
||||
pg.commit()
|
||||
log(" {} Issue Types created".format(types_created))
|
||||
|
||||
# 5. Import Tickets as Issues
|
||||
log("")
|
||||
log("--- Importing Tickets → Issues ---")
|
||||
|
||||
# Build message lookup by ticket_id
|
||||
msgs_by_ticket = {}
|
||||
for m in open_msgs:
|
||||
msgs_by_ticket.setdefault(m["ticket_id"], []).append(m)
|
||||
|
||||
# Track legacy_ticket_id → ERPNext issue name for parent mapping
|
||||
ticket_to_issue = {}
|
||||
i_ok = i_skip = i_err = 0
|
||||
comm_ok = 0
|
||||
|
||||
for i, t in enumerate(tickets):
|
||||
tid = t["id"]
|
||||
if tid in existing_issues:
|
||||
i_skip += 1
|
||||
continue
|
||||
|
||||
subject = clean(t.get("subject")) or "Ticket #{}".format(tid)
|
||||
status = STATUS_MAP.get(t.get("status", "open"), "Open")
|
||||
priority = PRIORITY_MAP.get(t.get("priority", 2), "Medium")
|
||||
dept_name = dept_map.get(t.get("dept_id"), None)
|
||||
cust_name = cust_map.get(t.get("account_id"))
|
||||
|
||||
opening_date = ts_to_dateonly(t.get("date_create"))
|
||||
opening_time = None
|
||||
if t.get("date_create") and t["date_create"] > 0:
|
||||
try:
|
||||
opening_time = datetime.fromtimestamp(int(t["date_create"]), tz=timezone.utc).strftime("%H:%M:%S")
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
# Is this an incident (parent)?
|
||||
is_incident = 0
|
||||
# Check if other tickets reference this one as parent
|
||||
# We'll do a second pass for this
|
||||
|
||||
issue_name = uid("ISS-")
|
||||
|
||||
try:
|
||||
pgc.execute("""
|
||||
INSERT INTO "tabIssue" (
|
||||
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||
naming_series, subject, status, priority, issue_type,
|
||||
customer, company, opening_date, opening_time,
|
||||
legacy_ticket_id, is_incident,
|
||||
parent_incident, service_location
|
||||
) VALUES (
|
||||
%s, %s, %s, %s, %s, 0, 0,
|
||||
'ISS-.YYYY.-', %s, %s, %s, %s,
|
||||
%s, %s, %s, %s,
|
||||
%s, %s, %s, %s
|
||||
)
|
||||
""", (issue_name, ts, ts, ADMIN, ADMIN,
|
||||
subject[:255], status, priority, dept_name,
|
||||
cust_name, COMPANY, opening_date, opening_time,
|
||||
tid, 0, None, None))
|
||||
|
||||
ticket_to_issue[tid] = issue_name
|
||||
i_ok += 1
|
||||
|
||||
# Import messages for open/pending tickets
|
||||
for m in msgs_by_ticket.get(tid, []):
|
||||
msg_text = m.get("msg") or ""
|
||||
if not msg_text.strip():
|
||||
continue
|
||||
sender = staff_to_user.get(m.get("staff_id"), ADMIN)
|
||||
msg_date = ts_to_date(m.get("date_orig"))
|
||||
|
||||
comm_name = uid("COM-")
|
||||
pgc.execute("""
|
||||
INSERT INTO "tabCommunication" (
|
||||
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||
subject, content, communication_type, comment_type,
|
||||
reference_doctype, reference_name,
|
||||
sender, communication_date, sent_or_received
|
||||
) VALUES (
|
||||
%s, %s, %s, %s, %s, 0, 0,
|
||||
%s, %s, 'Communication', 'Comment',
|
||||
'Issue', %s,
|
||||
%s, %s, 'Sent'
|
||||
)
|
||||
""", (comm_name, ts, ts, ADMIN, ADMIN,
|
||||
subject[:255], msg_text,
|
||||
issue_name,
|
||||
sender, msg_date or ts))
|
||||
comm_ok += 1
|
||||
|
||||
except Exception as e:
|
||||
i_err += 1
|
||||
pg.rollback()
|
||||
if i_err <= 20:
|
||||
log(" ERR ticket#{} -> {}".format(tid, str(e)[:100]))
|
||||
continue
|
||||
|
||||
if i_ok % 2000 == 0:
|
||||
pg.commit()
|
||||
log(" [{}/{}] issues={} comm={} skip={} err={}".format(
|
||||
i+1, len(tickets), i_ok, comm_ok, i_skip, i_err))
|
||||
|
||||
pg.commit()
|
||||
|
||||
# 6. Second pass: set parent_incident for sub-tickets
|
||||
log("")
|
||||
log("--- Setting parent/child relationships ---")
|
||||
parent_set = 0
|
||||
incident_set = 0
|
||||
|
||||
for t in tickets:
|
||||
if t.get("parent") and t["parent"] > 0:
|
||||
child_issue = ticket_to_issue.get(t["id"])
|
||||
parent_issue = ticket_to_issue.get(t["parent"])
|
||||
if child_issue and parent_issue:
|
||||
pgc.execute("""
|
||||
UPDATE "tabIssue" SET parent_incident = %s WHERE name = %s
|
||||
""", (parent_issue, child_issue))
|
||||
parent_set += 1
|
||||
|
||||
# Mark parent as incident
|
||||
pgc.execute("""
|
||||
UPDATE "tabIssue" SET is_incident = 1 WHERE name = %s AND is_incident = 0
|
||||
""", (parent_issue,))
|
||||
incident_set += 1
|
||||
|
||||
# Update affected_clients count for incidents
|
||||
pgc.execute("""
|
||||
UPDATE "tabIssue" i SET affected_clients = (
|
||||
SELECT COUNT(*) FROM "tabIssue" child WHERE child.parent_incident = i.name
|
||||
) WHERE i.is_incident = 1
|
||||
""")
|
||||
|
||||
pg.commit()
|
||||
pg.close()
|
||||
|
||||
log(" {} parent links set, {} incidents identified".format(parent_set, incident_set))
|
||||
|
||||
log("")
|
||||
log("=" * 60)
|
||||
log("Issue Types: {} created".format(types_created))
|
||||
log("Issues: {} created, {} skipped, {} errors".format(i_ok, i_skip, i_err))
|
||||
log("Communications: {} (open/pending tickets only)".format(comm_ok))
|
||||
log("Parent/child: {} links, {} incidents".format(parent_set, incident_set))
|
||||
log("Closed msgs: {} deferred (import separately if needed)".format(closed_msg_count))
|
||||
log("=" * 60)
|
||||
log("")
|
||||
log("Next: bench --site erp.gigafibre.ca clear-cache")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user