feat: subscription reimport, customer/doctype ID rename, zero-padded format
- Switch Ops data source from Subscription to Service Subscription (source of truth)
- Reimport 39,630 native Subscriptions from Service Subscription data
- Rename 15,302 customers to CUST-{legacy_customer_id} (eliminates hex UUIDs)
- Rename all doctypes to zero-padded 10-digit numeric format:
SINV-0000001234, PE-0000001234, ISS-0000001234, LOC-0000001234,
EQP-0000001234, SUB-0000001234, ASUB-0000001234
- Fix subscription pricing: LPB4 now correctly shows 0$/month
- Update ASUB- prefix detection in useSubscriptionActions.js
- Add reconciliation, reimport, and rename migration scripts
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
louispaulb
parent
7d7b4fdb06
commit
4a8718f67c
|
|
@ -8,12 +8,32 @@ import { authFetch } from 'src/api/auth'
|
||||||
import { BASE_URL } from 'src/config/erpnext'
|
import { BASE_URL } from 'src/config/erpnext'
|
||||||
import { formatMoney } from 'src/composables/useFormatters'
|
import { formatMoney } from 'src/composables/useFormatters'
|
||||||
|
|
||||||
// Frappe REST update for Subscription doctype
|
// Frappe REST update — supports both Service Subscription (legacy) and native Subscription
|
||||||
async function updateSub (name, fields) {
|
async function updateSub (name, fields) {
|
||||||
const res = await authFetch(BASE_URL + '/api/resource/Subscription/' + encodeURIComponent(name), {
|
// Detect which doctype based on name pattern: SUB-* = Service Subscription, ASUB-* = native Subscription
|
||||||
|
const doctype = name.startsWith('ASUB-') ? 'Subscription' : 'Service Subscription'
|
||||||
|
|
||||||
|
// Map normalized field names back to target doctype fields
|
||||||
|
const mapped = {}
|
||||||
|
for (const [k, v] of Object.entries(fields)) {
|
||||||
|
if (doctype === 'Service Subscription') {
|
||||||
|
// Map template field names → Service Subscription field names
|
||||||
|
if (k === 'actual_price') mapped.monthly_price = v
|
||||||
|
else if (k === 'custom_description') mapped.plan_name = v
|
||||||
|
else if (k === 'billing_frequency') mapped.billing_cycle = v === 'A' ? 'Annuel' : 'Mensuel'
|
||||||
|
else if (k === 'status') mapped.status = v === 'Active' ? 'Actif' : v === 'Cancelled' ? 'Annulé' : v
|
||||||
|
else if (k === 'cancelation_date') mapped.cancellation_date = v
|
||||||
|
else if (k === 'cancel_at_period_end') { /* not applicable to Service Subscription */ }
|
||||||
|
else mapped[k] = v
|
||||||
|
} else {
|
||||||
|
mapped[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await authFetch(BASE_URL + '/api/resource/' + encodeURIComponent(doctype) + '/' + encodeURIComponent(name), {
|
||||||
method: 'PUT',
|
method: 'PUT',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify(fields),
|
body: JSON.stringify(mapped),
|
||||||
})
|
})
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const err = await res.text()
|
const err = await res.text()
|
||||||
|
|
|
||||||
|
|
@ -22,33 +22,50 @@ export function subSection (sub) {
|
||||||
const price = parseFloat(sub.actual_price || 0)
|
const price = parseFloat(sub.actual_price || 0)
|
||||||
const group = (sub.item_group || '').trim()
|
const group = (sub.item_group || '').trim()
|
||||||
const sku = (sub.item_code || '').toUpperCase()
|
const sku = (sub.item_code || '').toUpperCase()
|
||||||
|
const label = (sub.custom_description || sub.item_name || sub.plan_name || '').toLowerCase()
|
||||||
|
|
||||||
if (price < 0 || sku.startsWith('RAB')) return 'Rabais'
|
// Negative price = rabais
|
||||||
|
if (price < 0 || sku.startsWith('RAB') || label.includes('rabais')) return 'Rabais'
|
||||||
|
|
||||||
|
// Match by item_group / service_category
|
||||||
for (const [section, cfg] of Object.entries(SECTION_MAP)) {
|
for (const [section, cfg] of Object.entries(SECTION_MAP)) {
|
||||||
if (section === 'Rabais') continue
|
if (section === 'Rabais') continue
|
||||||
if (cfg.match.some(m => group.includes(m))) return section
|
if (cfg.match.some(m => group.includes(m))) return section
|
||||||
}
|
}
|
||||||
|
|
||||||
// SKU-based fallback
|
// Category-based matching (from Service Subscription.service_category)
|
||||||
|
if (group === 'Internet') return 'Internet'
|
||||||
|
if (group === 'VoIP' || group === 'Téléphonie') return 'Téléphonie'
|
||||||
|
if (group === 'IPTV' || group === 'Télévision') return 'Télévision'
|
||||||
|
if (group === 'Hébergement') return 'Hébergement'
|
||||||
|
if (group === 'Bundle') return 'Internet'
|
||||||
|
|
||||||
|
// SKU-based fallback (works for both legacy item_codes and SUB-* names)
|
||||||
if (/^(FTTH|FTTB|TURBO|FIBRE|FORFERF|FORFBASE|FORFPERF|FORFPOP|FORFMES|SYM|VIP|ENT|COM|FIBCOM|HV)/.test(sku)) return 'Internet'
|
if (/^(FTTH|FTTB|TURBO|FIBRE|FORFERF|FORFBASE|FORFPERF|FORFPOP|FORFMES|SYM|VIP|ENT|COM|FIBCOM|HV)/.test(sku)) return 'Internet'
|
||||||
if (/^(TELEP|FAX|SERV911|SERVTEL|TELE_)/.test(sku)) return 'Téléphonie'
|
if (/^(TELEP|FAX|SERV911|SERVTEL|TELE_)/.test(sku)) return 'Téléphonie'
|
||||||
if (/^(TV|STB|RABTV)/.test(sku)) return 'Télévision'
|
if (/^(TV|STB|RABTV)/.test(sku)) return 'Télévision'
|
||||||
if (/^(LOC|FTT_H|FTTH_LOC)/.test(sku)) return 'Équipement'
|
if (/^(LOC|FTT_H|FTTH_LOC)/.test(sku)) return 'Équipement'
|
||||||
if (/^(HEB|DOM)/.test(sku)) return 'Hébergement'
|
if (/^(HEB|DOM)/.test(sku)) return 'Hébergement'
|
||||||
|
|
||||||
|
// Label-based fallback (Service Subscription plan_name)
|
||||||
|
if (/fibre|internet|giga|illimit|turbo|optimum/i.test(label)) return 'Internet'
|
||||||
|
if (/téléphon|voip|phone|fax|911/i.test(label)) return 'Téléphonie'
|
||||||
|
if (/télé|tv|décodeur|stb/i.test(label)) return 'Télévision'
|
||||||
|
if (/location|modem|routeur|appareil|frais.*accès/i.test(label)) return 'Équipement'
|
||||||
|
if (/héberg|domaine|cloud|espace/i.test(label)) return 'Hébergement'
|
||||||
|
|
||||||
return 'Autre'
|
return 'Autre'
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isRebate (sub) {
|
export function isRebate (sub) {
|
||||||
return parseFloat(sub.actual_price || 0) < 0
|
return parseFloat(sub.actual_price || sub.monthly_price || 0) < 0
|
||||||
}
|
}
|
||||||
|
|
||||||
export function subMainLabel (sub) {
|
export function subMainLabel (sub) {
|
||||||
if (sub.custom_description && sub.custom_description.trim()) {
|
if (sub.custom_description && sub.custom_description.trim()) {
|
||||||
return sub.custom_description
|
return sub.custom_description
|
||||||
}
|
}
|
||||||
return sub.item_name || sub.item_code || sub.name
|
return sub.item_name || sub.plan_name || sub.item_code || sub.name
|
||||||
}
|
}
|
||||||
|
|
||||||
export function subSubLabel (sub) {
|
export function subSubLabel (sub) {
|
||||||
|
|
|
||||||
|
|
@ -126,9 +126,9 @@
|
||||||
<div class="sub-row clickable-row" :class="{ 'sub-rebate': isRebate(sub), 'sub-cancelled': sub.status === 'Cancelled' }">
|
<div class="sub-row clickable-row" :class="{ 'sub-rebate': isRebate(sub), 'sub-cancelled': sub.status === 'Cancelled' }">
|
||||||
<div class="row items-center no-wrap">
|
<div class="row items-center no-wrap">
|
||||||
<q-icon name="drag_indicator" class="drag-handle text-grey-4 q-mr-xs" size="16px" style="cursor:grab" />
|
<q-icon name="drag_indicator" class="drag-handle text-grey-4 q-mr-xs" size="16px" style="cursor:grab" />
|
||||||
<div class="col" style="min-width:0" @click="openModal('Subscription', sub.name)">
|
<div class="col" style="min-width:0" @click="openModal('Service Subscription', sub.name)">
|
||||||
<div class="row items-center no-wrap q-gutter-x-sm">
|
<div class="row items-center no-wrap q-gutter-x-sm">
|
||||||
<code class="sub-sku">{{ sub.item_code || '—' }}</code>
|
<code class="sub-sku">{{ sub.name }}</code>
|
||||||
<span class="text-weight-medium ellipsis-text" style="flex:1;min-width:0">
|
<span class="text-weight-medium ellipsis-text" style="flex:1;min-width:0">
|
||||||
{{ subMainLabel(sub) }}
|
{{ subMainLabel(sub) }}
|
||||||
</span>
|
</span>
|
||||||
|
|
@ -172,8 +172,8 @@
|
||||||
class="sub-row clickable-row" :class="{ 'sub-rebate': isRebate(sub), 'sub-cancelled': sub.status === 'Cancelled' }">
|
class="sub-row clickable-row" :class="{ 'sub-rebate': isRebate(sub), 'sub-cancelled': sub.status === 'Cancelled' }">
|
||||||
<div class="row items-center no-wrap">
|
<div class="row items-center no-wrap">
|
||||||
<div class="col" style="min-width:0">
|
<div class="col" style="min-width:0">
|
||||||
<div class="row items-center no-wrap q-gutter-x-sm" @click="openModal('Subscription', sub.name)">
|
<div class="row items-center no-wrap q-gutter-x-sm" @click="openModal('Service Subscription', sub.name)">
|
||||||
<code class="sub-sku">{{ sub.item_code || '—' }}</code>
|
<code class="sub-sku">{{ sub.name }}</code>
|
||||||
<span class="sub-freq annual">A</span>
|
<span class="sub-freq annual">A</span>
|
||||||
<span class="text-weight-medium ellipsis-text" style="flex:1;min-width:0">
|
<span class="text-weight-medium ellipsis-text" style="flex:1;min-width:0">
|
||||||
{{ subMainLabel(sub) }}
|
{{ subMainLabel(sub) }}
|
||||||
|
|
@ -600,13 +600,26 @@ async function loadCustomer (id) {
|
||||||
'longitude', 'latitude'],
|
'longitude', 'latitude'],
|
||||||
limit: 100, orderBy: 'status asc, address_line asc',
|
limit: 100, orderBy: 'status asc, address_line asc',
|
||||||
}),
|
}),
|
||||||
listDocs('Subscription', {
|
listDocs('Service Subscription', {
|
||||||
filters: partyFilter,
|
filters: custFilter,
|
||||||
fields: ['name', 'status', 'start_date', 'service_location', 'radius_user',
|
fields: ['name', 'status', 'start_date', 'end_date', 'service_location',
|
||||||
'actual_price', 'custom_description', 'item_code', 'item_group', 'billing_frequency', 'item_name',
|
'monthly_price', 'plan_name', 'service_category', 'billing_cycle',
|
||||||
'cancel_at_period_end', 'current_invoice_start', 'current_invoice_end', 'end_date', 'cancelation_date'],
|
'speed_down', 'speed_up', 'cancellation_date', 'cancellation_reason', 'notes'],
|
||||||
limit: 100, orderBy: 'start_date desc',
|
limit: 200, orderBy: 'start_date desc',
|
||||||
}),
|
}).then(subs => subs.map(s => ({
|
||||||
|
// Normalize to match template field names (formerly from Subscription doctype)
|
||||||
|
...s,
|
||||||
|
actual_price: s.monthly_price,
|
||||||
|
custom_description: s.plan_name,
|
||||||
|
item_name: s.plan_name,
|
||||||
|
item_code: s.name,
|
||||||
|
item_group: s.service_category || '',
|
||||||
|
billing_frequency: s.billing_cycle === 'Annuel' ? 'A' : 'M',
|
||||||
|
cancel_at_period_end: 0,
|
||||||
|
cancelation_date: s.cancellation_date,
|
||||||
|
// Map status: Actif→Active, Annulé→Cancelled, etc.
|
||||||
|
status: s.status === 'Actif' ? 'Active' : s.status === 'Annulé' ? 'Cancelled' : s.status === 'Suspendu' ? 'Cancelled' : s.status === 'En attente' ? 'Active' : s.status,
|
||||||
|
}))),
|
||||||
listDocs('Service Equipment', {
|
listDocs('Service Equipment', {
|
||||||
filters: custFilter,
|
filters: custFilter,
|
||||||
fields: ['name', 'equipment_type', 'brand', 'model', 'serial_number', 'mac_address',
|
fields: ['name', 'equipment_type', 'brand', 'model', 'serial_number', 'mac_address',
|
||||||
|
|
|
||||||
491
scripts/migration/reconcile_subscriptions.py
Normal file
491
scripts/migration/reconcile_subscriptions.py
Normal file
|
|
@ -0,0 +1,491 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Reconcile Service Subscription (complete legacy import) vs Subscription (ERPNext native).
|
||||||
|
|
||||||
|
Identifies per-customer discrepancies:
|
||||||
|
- Missing records in Subscription that exist in Service Subscription
|
||||||
|
- Price/status mismatches
|
||||||
|
- Generates a sync plan to fix the native Subscription table
|
||||||
|
|
||||||
|
Run inside erpnext-backend-1:
|
||||||
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reconcile_subscriptions.py
|
||||||
|
|
||||||
|
Modes:
|
||||||
|
--audit Audit only — report discrepancies (default)
|
||||||
|
--sync Create missing Subscriptions from Service Subscriptions
|
||||||
|
--fix Fix price/status mismatches on existing Subscriptions
|
||||||
|
--full Audit + Sync + Fix (do everything)
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
os.chdir("/home/frappe/frappe-bench/sites")
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
||||||
|
frappe.connect()
|
||||||
|
frappe.local.flags.ignore_permissions = True
|
||||||
|
print(f"Connected: {frappe.local.site}")
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
COMPANY = "TARGO"
|
||||||
|
TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T"
|
||||||
|
|
||||||
|
|
||||||
|
def uid(prefix=""):
|
||||||
|
return prefix + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 1: Load all Service Subscriptions (source of truth)
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def load_service_subscriptions():
|
||||||
|
"""Load all Service Subscriptions grouped by customer."""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("LOADING SERVICE SUBSCRIPTIONS (source of truth)")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
rows = frappe.db.sql("""
|
||||||
|
SELECT name, customer, service_location, plan_name, monthly_price,
|
||||||
|
status, service_category, billing_cycle, start_date, end_date,
|
||||||
|
speed_down, speed_up, cancellation_date, cancellation_reason,
|
||||||
|
legacy_service_id, product_sku, radius_user, radius_password
|
||||||
|
FROM "tabService Subscription"
|
||||||
|
ORDER BY customer, service_location, monthly_price DESC
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
by_customer = {}
|
||||||
|
for r in rows:
|
||||||
|
cust = r["customer"]
|
||||||
|
if cust not in by_customer:
|
||||||
|
by_customer[cust] = []
|
||||||
|
by_customer[cust].append(r)
|
||||||
|
|
||||||
|
total = len(rows)
|
||||||
|
active = sum(1 for r in rows if r["status"] == "Actif")
|
||||||
|
customers = len(by_customer)
|
||||||
|
print(f" Total: {total} records, {active} active, {customers} customers")
|
||||||
|
return by_customer
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 2: Load all Subscriptions (ERPNext native)
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def load_subscriptions():
|
||||||
|
"""Load all native Subscriptions grouped by party (customer)."""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("LOADING SUBSCRIPTIONS (ERPNext native)")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
rows = frappe.db.sql("""
|
||||||
|
SELECT s.name, s.party as customer, s.status, s.start_date, s.end_date,
|
||||||
|
s.actual_price, s.custom_description, s.item_code, s.item_group,
|
||||||
|
s.billing_frequency, s.legacy_service_id, s.service_location,
|
||||||
|
s.radius_user, s.cancel_at_period_end, s.cancelation_date,
|
||||||
|
s.additional_discount_amount
|
||||||
|
FROM "tabSubscription" s
|
||||||
|
ORDER BY s.party, s.service_location, s.actual_price DESC
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
by_customer = {}
|
||||||
|
by_legacy_id = {}
|
||||||
|
for r in rows:
|
||||||
|
cust = r["customer"]
|
||||||
|
if cust not in by_customer:
|
||||||
|
by_customer[cust] = []
|
||||||
|
by_customer[cust].append(r)
|
||||||
|
if r.get("legacy_service_id"):
|
||||||
|
by_legacy_id[r["legacy_service_id"]] = r
|
||||||
|
|
||||||
|
total = len(rows)
|
||||||
|
active = sum(1 for r in rows if r["status"] == "Active")
|
||||||
|
customers = len(by_customer)
|
||||||
|
print(f" Total: {total} records, {active} active, {customers} customers")
|
||||||
|
return by_customer, by_legacy_id
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 3: Audit — Compare per-customer
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def audit(ss_by_cust, sub_by_cust, sub_by_legacy_id):
|
||||||
|
"""Compare per-customer and identify discrepancies."""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("AUDIT: COMPARING PER-CUSTOMER")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
all_customers = set(list(ss_by_cust.keys()) + list(sub_by_cust.keys()))
|
||||||
|
|
||||||
|
# Discrepancy tracking
|
||||||
|
missing_in_sub = [] # Service Subscriptions with no matching Subscription
|
||||||
|
price_mismatches = [] # Matching records with different prices
|
||||||
|
status_mismatches = [] # Matching records with different statuses
|
||||||
|
count_mismatches = [] # Customers with different record counts
|
||||||
|
total_mismatches = [] # Customers with different monthly totals
|
||||||
|
|
||||||
|
customers_ok = 0
|
||||||
|
customers_mismatch = 0
|
||||||
|
|
||||||
|
for cust in sorted(all_customers):
|
||||||
|
ss_list = ss_by_cust.get(cust, [])
|
||||||
|
sub_list = sub_by_cust.get(cust, [])
|
||||||
|
|
||||||
|
# Compare totals
|
||||||
|
ss_total = sum(float(s.get("monthly_price") or 0) for s in ss_list if s["status"] == "Actif")
|
||||||
|
sub_total = sum(float(s.get("actual_price") or 0) for s in sub_list if s["status"] == "Active")
|
||||||
|
|
||||||
|
ss_active = [s for s in ss_list if s["status"] == "Actif"]
|
||||||
|
sub_active = [s for s in sub_list if s["status"] == "Active"]
|
||||||
|
|
||||||
|
has_issue = False
|
||||||
|
|
||||||
|
# Count mismatch
|
||||||
|
if len(ss_active) != len(sub_active):
|
||||||
|
count_mismatches.append({
|
||||||
|
"customer": cust,
|
||||||
|
"ss_count": len(ss_active),
|
||||||
|
"sub_count": len(sub_active),
|
||||||
|
"diff": len(ss_active) - len(sub_active),
|
||||||
|
})
|
||||||
|
has_issue = True
|
||||||
|
|
||||||
|
# Total mismatch (more than 0.01$ difference)
|
||||||
|
if abs(ss_total - sub_total) > 0.01:
|
||||||
|
total_mismatches.append({
|
||||||
|
"customer": cust,
|
||||||
|
"ss_total": ss_total,
|
||||||
|
"sub_total": sub_total,
|
||||||
|
"diff": ss_total - sub_total,
|
||||||
|
})
|
||||||
|
has_issue = True
|
||||||
|
|
||||||
|
# Per-record matching via legacy_service_id
|
||||||
|
for ss in ss_list:
|
||||||
|
legacy_id = ss.get("legacy_service_id")
|
||||||
|
if legacy_id and legacy_id in sub_by_legacy_id:
|
||||||
|
sub = sub_by_legacy_id[legacy_id]
|
||||||
|
# Price check
|
||||||
|
ss_price = float(ss.get("monthly_price") or 0)
|
||||||
|
sub_price = float(sub.get("actual_price") or 0)
|
||||||
|
if abs(ss_price - sub_price) > 0.01:
|
||||||
|
price_mismatches.append({
|
||||||
|
"customer": cust,
|
||||||
|
"ss_name": ss["name"],
|
||||||
|
"sub_name": sub["name"],
|
||||||
|
"legacy_id": legacy_id,
|
||||||
|
"plan": ss.get("plan_name") or "",
|
||||||
|
"ss_price": ss_price,
|
||||||
|
"sub_price": sub_price,
|
||||||
|
"diff": ss_price - sub_price,
|
||||||
|
})
|
||||||
|
has_issue = True
|
||||||
|
|
||||||
|
# Status check
|
||||||
|
ss_status = "Active" if ss["status"] == "Actif" else "Cancelled"
|
||||||
|
sub_status = sub["status"]
|
||||||
|
if ss_status != sub_status:
|
||||||
|
status_mismatches.append({
|
||||||
|
"customer": cust,
|
||||||
|
"ss_name": ss["name"],
|
||||||
|
"sub_name": sub["name"],
|
||||||
|
"legacy_id": legacy_id,
|
||||||
|
"ss_status": ss["status"],
|
||||||
|
"sub_status": sub_status,
|
||||||
|
})
|
||||||
|
has_issue = True
|
||||||
|
elif legacy_id:
|
||||||
|
# No matching Subscription found
|
||||||
|
missing_in_sub.append({
|
||||||
|
"customer": cust,
|
||||||
|
"ss_name": ss["name"],
|
||||||
|
"legacy_id": legacy_id,
|
||||||
|
"plan_name": ss.get("plan_name") or "",
|
||||||
|
"monthly_price": float(ss.get("monthly_price") or 0),
|
||||||
|
"status": ss["status"],
|
||||||
|
"service_location": ss.get("service_location") or "",
|
||||||
|
"service_category": ss.get("service_category") or "",
|
||||||
|
"billing_cycle": ss.get("billing_cycle") or "Mensuel",
|
||||||
|
"start_date": ss.get("start_date"),
|
||||||
|
"product_sku": ss.get("product_sku") or "",
|
||||||
|
"radius_user": ss.get("radius_user") or "",
|
||||||
|
"radius_password": ss.get("radius_password") or "",
|
||||||
|
})
|
||||||
|
has_issue = True
|
||||||
|
|
||||||
|
if has_issue:
|
||||||
|
customers_mismatch += 1
|
||||||
|
else:
|
||||||
|
customers_ok += 1
|
||||||
|
|
||||||
|
# ── Report ──
|
||||||
|
print(f"\n Customers OK (totals match): {customers_ok}")
|
||||||
|
print(f" Customers with discrepancies: {customers_mismatch}")
|
||||||
|
print(f" ---")
|
||||||
|
print(f" Missing in Subscription: {len(missing_in_sub)}")
|
||||||
|
print(f" Price mismatches: {len(price_mismatches)}")
|
||||||
|
print(f" Status mismatches: {len(status_mismatches)}")
|
||||||
|
print(f" Count mismatches: {len(count_mismatches)}")
|
||||||
|
print(f" Total (monthly $) mismatches: {len(total_mismatches)}")
|
||||||
|
|
||||||
|
# Top 20 total mismatches
|
||||||
|
if total_mismatches:
|
||||||
|
print(f"\n TOP 20 MONTHLY TOTAL MISMATCHES:")
|
||||||
|
sorted_tm = sorted(total_mismatches, key=lambda x: abs(x["diff"]), reverse=True)
|
||||||
|
for tm in sorted_tm[:20]:
|
||||||
|
print(f" {tm['customer']:25s} SS: {tm['ss_total']:>8.2f} SUB: {tm['sub_total']:>8.2f} DIFF: {tm['diff']:>+8.2f}")
|
||||||
|
|
||||||
|
# Top 20 count mismatches
|
||||||
|
if count_mismatches:
|
||||||
|
print(f"\n TOP 20 COUNT MISMATCHES:")
|
||||||
|
sorted_cm = sorted(count_mismatches, key=lambda x: abs(x["diff"]), reverse=True)
|
||||||
|
for cm in sorted_cm[:20]:
|
||||||
|
print(f" {cm['customer']:25s} SS: {cm['ss_count']:3d} SUB: {cm['sub_count']:3d} DIFF: {cm['diff']:+3d}")
|
||||||
|
|
||||||
|
# Missing by category
|
||||||
|
if missing_in_sub:
|
||||||
|
cats = {}
|
||||||
|
for m in missing_in_sub:
|
||||||
|
cat = m["service_category"] or "Unknown"
|
||||||
|
cats[cat] = cats.get(cat, 0) + 1
|
||||||
|
print(f"\n MISSING IN SUBSCRIPTION BY CATEGORY:")
|
||||||
|
for cat, count in sorted(cats.items(), key=lambda x: -x[1]):
|
||||||
|
print(f" {cat:25s} {count:5d}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"missing": missing_in_sub,
|
||||||
|
"price_mismatches": price_mismatches,
|
||||||
|
"status_mismatches": status_mismatches,
|
||||||
|
"count_mismatches": count_mismatches,
|
||||||
|
"total_mismatches": total_mismatches,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 4: Sync — Create missing Subscriptions
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def sync_missing(missing_records):
|
||||||
|
"""Create native Subscriptions for records missing from the Subscription table."""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print(f"SYNC: CREATING {len(missing_records)} MISSING SUBSCRIPTIONS")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
if not missing_records:
|
||||||
|
print(" Nothing to sync!")
|
||||||
|
return
|
||||||
|
|
||||||
|
ts = now()
|
||||||
|
|
||||||
|
# Load existing subscription plans
|
||||||
|
plans = frappe.db.sql('SELECT plan_name, name FROM "tabSubscription Plan"', as_dict=True)
|
||||||
|
plan_map = {p["plan_name"]: p["name"] for p in plans}
|
||||||
|
|
||||||
|
# Load existing items (to find or create plans)
|
||||||
|
items = frappe.db.sql('SELECT name, item_name, item_group FROM "tabItem"', as_dict=True)
|
||||||
|
item_map = {i["name"]: i for i in items}
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
errors = 0
|
||||||
|
plan_created = 0
|
||||||
|
|
||||||
|
for rec in missing_records:
|
||||||
|
sku = rec["product_sku"] or "UNKNOWN"
|
||||||
|
plan_name = f"PLAN-{sku}"
|
||||||
|
|
||||||
|
# Create plan if it doesn't exist
|
||||||
|
if plan_name not in plan_map:
|
||||||
|
plan_id = uid("SP-")
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription Plan" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
plan_name, item, currency, price_determination, cost,
|
||||||
|
billing_interval, billing_interval_count
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
%s, %s, 'CAD', 'Fixed Rate', %s, %s, 1)
|
||||||
|
""", (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku,
|
||||||
|
abs(rec["monthly_price"]),
|
||||||
|
"Year" if rec["billing_cycle"] == "Annuel" else "Month"))
|
||||||
|
plan_map[plan_name] = plan_id
|
||||||
|
plan_created += 1
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
print(f" ERR plan {plan_name}: {str(e)[:80]}")
|
||||||
|
|
||||||
|
# Determine status
|
||||||
|
status = "Active" if rec["status"] == "Actif" else "Cancelled"
|
||||||
|
|
||||||
|
# Determine discount: negative price = discount record, handle differently
|
||||||
|
price = rec["monthly_price"]
|
||||||
|
discount = 0
|
||||||
|
if price < 0:
|
||||||
|
# This is a rebate/discount line — store as negative actual_price
|
||||||
|
discount = 0 # Don't use additional_discount for negative-price lines
|
||||||
|
|
||||||
|
# Start date
|
||||||
|
start_date = rec.get("start_date") or "2020-01-01"
|
||||||
|
if hasattr(start_date, 'strftime'):
|
||||||
|
start_date = start_date.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
# Billing frequency
|
||||||
|
billing_freq = "A" if rec["billing_cycle"] == "Annuel" else "M"
|
||||||
|
|
||||||
|
sub_id = uid("SUB-")
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
party_type, party, company, status,
|
||||||
|
start_date, generate_invoice_at, days_until_due,
|
||||||
|
follow_calendar_months, generate_new_invoices_past_due_date,
|
||||||
|
submit_invoice, cancel_at_period_end,
|
||||||
|
sales_tax_template,
|
||||||
|
additional_discount_amount,
|
||||||
|
radius_user, radius_pwd, legacy_service_id,
|
||||||
|
service_location,
|
||||||
|
actual_price, custom_description, item_code, item_group,
|
||||||
|
billing_frequency
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
'Customer', %s, %s, %s,
|
||||||
|
%s, 'Beginning of the current subscription period', 30,
|
||||||
|
0, 1, 0, 0,
|
||||||
|
%s, %s,
|
||||||
|
%s, %s, %s,
|
||||||
|
%s,
|
||||||
|
%s, %s, %s, %s, %s
|
||||||
|
)
|
||||||
|
""", (sub_id, ts, ts, ADMIN, ADMIN,
|
||||||
|
rec["customer"], COMPANY, status,
|
||||||
|
start_date,
|
||||||
|
TAX_TEMPLATE, discount,
|
||||||
|
rec.get("radius_user") or None,
|
||||||
|
rec.get("radius_password") or None,
|
||||||
|
rec["legacy_id"],
|
||||||
|
rec.get("service_location") or None,
|
||||||
|
price, rec.get("plan_name") or "",
|
||||||
|
sku, rec.get("service_category") or "",
|
||||||
|
billing_freq))
|
||||||
|
|
||||||
|
# Create plan detail child record
|
||||||
|
if plan_name in plan_map:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription Plan Detail" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
plan, qty,
|
||||||
|
parent, parentfield, parenttype
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 1,
|
||||||
|
%s, 1,
|
||||||
|
%s, 'plans', 'Subscription')
|
||||||
|
""", (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id))
|
||||||
|
|
||||||
|
created += 1
|
||||||
|
|
||||||
|
if created % 500 == 0:
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" [{created}/{len(missing_records)}] created...")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
errors += 1
|
||||||
|
frappe.db.rollback()
|
||||||
|
if errors <= 20:
|
||||||
|
print(f" ERR {rec['customer']} {rec['ss_name']}: {str(e)[:100]}")
|
||||||
|
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f"\n Plans created: {plan_created}")
|
||||||
|
print(f" Subscriptions created: {created}")
|
||||||
|
print(f" Errors: {errors}")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 5: Fix — Correct price/status mismatches
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def fix_mismatches(price_mismatches, status_mismatches):
|
||||||
|
"""Fix price and status mismatches on existing Subscriptions."""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print(f"FIX: CORRECTING {len(price_mismatches)} PRICE + {len(status_mismatches)} STATUS MISMATCHES")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
fixed_price = 0
|
||||||
|
fixed_status = 0
|
||||||
|
|
||||||
|
for pm in price_mismatches:
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE "tabSubscription"
|
||||||
|
SET actual_price = %s, modified = %s
|
||||||
|
WHERE name = %s
|
||||||
|
""", (pm["ss_price"], now(), pm["sub_name"]))
|
||||||
|
fixed_price += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ERR price fix {pm['sub_name']}: {str(e)[:80]}")
|
||||||
|
|
||||||
|
for sm in status_mismatches:
|
||||||
|
new_status = "Active" if sm["ss_status"] == "Actif" else "Cancelled"
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE "tabSubscription"
|
||||||
|
SET status = %s, modified = %s
|
||||||
|
WHERE name = %s
|
||||||
|
""", (new_status, now(), sm["sub_name"]))
|
||||||
|
fixed_status += 1
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ERR status fix {sm['sub_name']}: {str(e)[:80]}")
|
||||||
|
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Price fixes: {fixed_price}")
|
||||||
|
print(f" Status fixes: {fixed_status}")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# MAIN
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
def main():
|
||||||
|
mode = sys.argv[1] if len(sys.argv) > 1 else "--audit"
|
||||||
|
|
||||||
|
print(f"\nMode: {mode}")
|
||||||
|
print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||||
|
|
||||||
|
# Load data
|
||||||
|
ss_by_cust = load_service_subscriptions()
|
||||||
|
sub_by_cust, sub_by_legacy_id = load_subscriptions()
|
||||||
|
|
||||||
|
# Audit
|
||||||
|
results = audit(ss_by_cust, sub_by_cust, sub_by_legacy_id)
|
||||||
|
|
||||||
|
if mode in ("--sync", "--full"):
|
||||||
|
sync_missing(results["missing"])
|
||||||
|
|
||||||
|
if mode in ("--fix", "--full"):
|
||||||
|
fix_mismatches(results["price_mismatches"], results["status_mismatches"])
|
||||||
|
|
||||||
|
# Write report file
|
||||||
|
report_path = "/tmp/reconcile_report.json"
|
||||||
|
report = {
|
||||||
|
"timestamp": datetime.now().isoformat(),
|
||||||
|
"mode": mode,
|
||||||
|
"summary": {
|
||||||
|
"missing_in_subscription": len(results["missing"]),
|
||||||
|
"price_mismatches": len(results["price_mismatches"]),
|
||||||
|
"status_mismatches": len(results["status_mismatches"]),
|
||||||
|
"count_mismatches": len(results["count_mismatches"]),
|
||||||
|
"total_mismatches": len(results["total_mismatches"]),
|
||||||
|
},
|
||||||
|
"total_mismatches_top50": sorted(results["total_mismatches"], key=lambda x: abs(x["diff"]), reverse=True)[:50],
|
||||||
|
"missing_sample": results["missing"][:50],
|
||||||
|
}
|
||||||
|
with open(report_path, "w") as f:
|
||||||
|
json.dump(report, f, indent=2, default=str)
|
||||||
|
print(f"\nReport saved: {report_path}")
|
||||||
|
|
||||||
|
frappe.clear_cache()
|
||||||
|
print("\nDone — cache cleared")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
423
scripts/migration/reimport_subscriptions.py
Normal file
423
scripts/migration/reimport_subscriptions.py
Normal file
|
|
@ -0,0 +1,423 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Clean reimport of native Subscriptions from Service Subscription (source of truth).
|
||||||
|
|
||||||
|
Fixes the migration issue where Phase 3/4 created hex customer IDs (CUST-04ea550dcf)
|
||||||
|
instead of mapping to the correct sequential IDs (CUST-988).
|
||||||
|
|
||||||
|
Steps:
|
||||||
|
1. Purge all native Subscriptions + Plan Details
|
||||||
|
2. Recreate Subscriptions from Service Subscription data (correct customer mapping)
|
||||||
|
3. Delete orphaned hex customers (no remaining FK references)
|
||||||
|
4. Verify totals match
|
||||||
|
|
||||||
|
Run inside erpnext-backend-1:
|
||||||
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/reimport_subscriptions.py [--dry-run]
|
||||||
|
|
||||||
|
Log: stdout (run with tee)
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
os.chdir("/home/frappe/frappe-bench/sites")
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
||||||
|
frappe.connect()
|
||||||
|
frappe.local.flags.ignore_permissions = True
|
||||||
|
print(f"Connected: {frappe.local.site}")
|
||||||
|
|
||||||
|
DRY_RUN = "--dry-run" in sys.argv
|
||||||
|
if DRY_RUN:
|
||||||
|
print("*** DRY RUN — no changes will be written ***")
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
COMPANY = "TARGO"
|
||||||
|
TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T"
|
||||||
|
|
||||||
|
|
||||||
|
def uid(prefix=""):
|
||||||
|
return prefix + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 1: PURGE native Subscriptions
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 1: PURGE EXISTING SUBSCRIPTIONS")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0]
|
||||||
|
spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0]
|
||||||
|
sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0]
|
||||||
|
|
||||||
|
print(f" Current counts: {sub_count} Subscriptions, {spd_count} Plan Details, {sp_count} Plans")
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
frappe.db.sql('DELETE FROM "tabSubscription Plan Detail"')
|
||||||
|
frappe.db.sql('DELETE FROM "tabSubscription"')
|
||||||
|
frappe.db.sql('DELETE FROM "tabSubscription Plan"')
|
||||||
|
frappe.db.commit()
|
||||||
|
print(" Purged all Subscriptions, Plan Details, and Plans.")
|
||||||
|
else:
|
||||||
|
print(" [DRY RUN] Would purge all Subscriptions, Plan Details, and Plans.")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 2: LOAD Service Subscriptions (source of truth)
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 2: LOAD SERVICE SUBSCRIPTIONS")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
ss_rows = frappe.db.sql("""
|
||||||
|
SELECT name, customer, service_location, plan_name, monthly_price,
|
||||||
|
status, service_category, billing_cycle, start_date, end_date,
|
||||||
|
speed_down, speed_up, cancellation_date, cancellation_reason,
|
||||||
|
legacy_service_id, product_sku, radius_user, radius_password, notes
|
||||||
|
FROM "tabService Subscription"
|
||||||
|
ORDER BY customer, service_location, monthly_price DESC
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
total = len(ss_rows)
|
||||||
|
active = sum(1 for r in ss_rows if r["status"] == "Actif")
|
||||||
|
customers = len(set(r["customer"] for r in ss_rows))
|
||||||
|
print(f" Loaded {total} Service Subscriptions ({active} active) for {customers} customers")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 3: BUILD Subscription Plans from unique SKUs
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 3: CREATE SUBSCRIPTION PLANS")
|
||||||
|
print("=" * 70)
|
||||||
|
t0 = time.time()
|
||||||
|
|
||||||
|
# Collect unique SKUs with their typical price and category
|
||||||
|
sku_info = {}
|
||||||
|
for r in ss_rows:
|
||||||
|
sku = r.get("product_sku") or ""
|
||||||
|
if not sku:
|
||||||
|
continue
|
||||||
|
if sku not in sku_info:
|
||||||
|
sku_info[sku] = {
|
||||||
|
"price": abs(float(r["monthly_price"] or 0)),
|
||||||
|
"category": r.get("service_category") or "",
|
||||||
|
"billing_cycle": r.get("billing_cycle") or "Mensuel",
|
||||||
|
"plan_name": r.get("plan_name") or sku,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Also add a fallback plan for records without SKU
|
||||||
|
plan_map = {} # plan_name → plan_id
|
||||||
|
plans_created = 0
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
ts = now()
|
||||||
|
for sku, info in sku_info.items():
|
||||||
|
plan_name = f"PLAN-{sku}"
|
||||||
|
plan_id = uid("SP-")
|
||||||
|
billing_interval = "Year" if info["billing_cycle"] == "Annuel" else "Month"
|
||||||
|
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription Plan" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
plan_name, item, currency, price_determination, cost,
|
||||||
|
billing_interval, billing_interval_count
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
%s, %s, 'CAD', 'Fixed Rate', %s, %s, 1)
|
||||||
|
""", (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, info["price"], billing_interval))
|
||||||
|
plan_map[plan_name] = plan_id
|
||||||
|
plans_created += 1
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
print(f" ERR plan {sku}: {str(e)[:80]}")
|
||||||
|
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
print(f" Created {plans_created} Subscription Plans [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 4: CREATE Subscriptions from Service Subscriptions
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 4: CREATE SUBSCRIPTIONS")
|
||||||
|
print("=" * 70)
|
||||||
|
t0 = time.time()
|
||||||
|
|
||||||
|
# Verify all customers exist
|
||||||
|
all_customers = set(r["customer"] for r in ss_rows)
|
||||||
|
existing_customers = set(
|
||||||
|
r[0] for r in frappe.db.sql('SELECT name FROM "tabCustomer"')
|
||||||
|
)
|
||||||
|
missing_customers = all_customers - existing_customers
|
||||||
|
if missing_customers:
|
||||||
|
print(f" WARNING: {len(missing_customers)} customers referenced but don't exist!")
|
||||||
|
for c in list(missing_customers)[:10]:
|
||||||
|
print(f" {c}")
|
||||||
|
|
||||||
|
sub_ok = sub_err = 0
|
||||||
|
ts = now()
|
||||||
|
|
||||||
|
for i, ss in enumerate(ss_rows):
|
||||||
|
if DRY_RUN:
|
||||||
|
sub_ok += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cust = ss["customer"]
|
||||||
|
if cust in missing_customers:
|
||||||
|
sub_err += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Map status
|
||||||
|
if ss["status"] == "Actif":
|
||||||
|
status = "Active"
|
||||||
|
elif ss["status"] == "Annulé":
|
||||||
|
status = "Cancelled"
|
||||||
|
elif ss["status"] == "Suspendu":
|
||||||
|
status = "Past Due Date"
|
||||||
|
else:
|
||||||
|
status = "Active" # default
|
||||||
|
|
||||||
|
# Price
|
||||||
|
price = float(ss["monthly_price"] or 0)
|
||||||
|
|
||||||
|
# Start date
|
||||||
|
start_date = ss.get("start_date") or "2020-01-01"
|
||||||
|
if hasattr(start_date, "strftime"):
|
||||||
|
start_date = start_date.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
# End/cancellation date
|
||||||
|
end_date = ss.get("end_date")
|
||||||
|
if hasattr(end_date, "strftime"):
|
||||||
|
end_date = end_date.strftime("%Y-%m-%d")
|
||||||
|
cancel_date = ss.get("cancellation_date")
|
||||||
|
if hasattr(cancel_date, "strftime"):
|
||||||
|
cancel_date = cancel_date.strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
# Billing frequency
|
||||||
|
billing_freq = "A" if ss.get("billing_cycle") == "Annuel" else "M"
|
||||||
|
|
||||||
|
# SKU / plan
|
||||||
|
sku = ss.get("product_sku") or ""
|
||||||
|
plan_name = f"PLAN-{sku}" if sku else ""
|
||||||
|
|
||||||
|
# Category
|
||||||
|
category = ss.get("service_category") or ""
|
||||||
|
|
||||||
|
# Description: plan_name from SS
|
||||||
|
description = ss.get("plan_name") or ""
|
||||||
|
|
||||||
|
sub_id = uid("SUB-")
|
||||||
|
|
||||||
|
try:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
party_type, party, company, status,
|
||||||
|
start_date, generate_invoice_at, days_until_due,
|
||||||
|
follow_calendar_months, generate_new_invoices_past_due_date,
|
||||||
|
submit_invoice, cancel_at_period_end,
|
||||||
|
sales_tax_template,
|
||||||
|
additional_discount_amount,
|
||||||
|
radius_user, radius_pwd, legacy_service_id,
|
||||||
|
service_location,
|
||||||
|
actual_price, custom_description, item_code, item_group,
|
||||||
|
billing_frequency,
|
||||||
|
cancelation_date
|
||||||
|
) VALUES (
|
||||||
|
%s, %s, %s, %s, %s, 0, 0,
|
||||||
|
'Customer', %s, %s, %s,
|
||||||
|
%s, 'Beginning of the current subscription period', 30,
|
||||||
|
1, 1, 0, 0,
|
||||||
|
%s, 0,
|
||||||
|
%s, %s, %s,
|
||||||
|
%s,
|
||||||
|
%s, %s, %s, %s, %s,
|
||||||
|
%s
|
||||||
|
)
|
||||||
|
""", (sub_id, ts, ts, ADMIN, ADMIN,
|
||||||
|
cust, COMPANY, status,
|
||||||
|
start_date,
|
||||||
|
TAX_TEMPLATE,
|
||||||
|
ss.get("radius_user") or None,
|
||||||
|
ss.get("radius_password") or None,
|
||||||
|
ss.get("legacy_service_id"),
|
||||||
|
ss.get("service_location") or None,
|
||||||
|
price, description, sku, category, billing_freq,
|
||||||
|
cancel_date))
|
||||||
|
|
||||||
|
# Create plan detail child record
|
||||||
|
if plan_name and plan_name in plan_map:
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSubscription Plan Detail" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
plan, qty,
|
||||||
|
parent, parentfield, parenttype
|
||||||
|
) VALUES (%s, %s, %s, %s, %s, 0, 1,
|
||||||
|
%s, 1,
|
||||||
|
%s, 'plans', 'Subscription')
|
||||||
|
""", (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id))
|
||||||
|
|
||||||
|
sub_ok += 1
|
||||||
|
|
||||||
|
if sub_ok % 2000 == 0:
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" [{sub_ok}/{total}] created... [{time.time()-t0:.0f}s]")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
sub_err += 1
|
||||||
|
frappe.db.rollback()
|
||||||
|
if sub_err <= 20:
|
||||||
|
print(f" ERR {cust}/{ss['name']}: {str(e)[:100]}")
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
print(f" Created {sub_ok} Subscriptions, {sub_err} errors [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 5: CLEANUP — Delete orphaned hex customers
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 5: CLEANUP ORPHANED HEX CUSTOMERS")
|
||||||
|
print("=" * 70)
|
||||||
|
t0 = time.time()
|
||||||
|
|
||||||
|
# Find hex-pattern customers (non-numeric suffix after CUST-)
|
||||||
|
hex_customers = frappe.db.sql("""
|
||||||
|
SELECT name, customer_name, legacy_account_id
|
||||||
|
FROM "tabCustomer"
|
||||||
|
WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
print(f" Found {len(hex_customers)} hex-pattern customers")
|
||||||
|
|
||||||
|
if hex_customers:
|
||||||
|
# Check which hex customers still have FK references using subqueries (avoid ANY() limits)
|
||||||
|
hex_subquery = """(SELECT name FROM "tabCustomer" WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$')"""
|
||||||
|
|
||||||
|
fk_tables = [
|
||||||
|
("tabSales Invoice", "customer"),
|
||||||
|
("tabPayment Entry", "party"),
|
||||||
|
("tabIssue", "customer"),
|
||||||
|
("tabService Location", "customer"),
|
||||||
|
("tabService Subscription", "customer"),
|
||||||
|
("tabSubscription", "party"),
|
||||||
|
("tabService Equipment", "customer"),
|
||||||
|
]
|
||||||
|
|
||||||
|
referenced = set()
|
||||||
|
for table, col in fk_tables:
|
||||||
|
try:
|
||||||
|
refs = frappe.db.sql(
|
||||||
|
f'SELECT DISTINCT "{col}" FROM "{table}" WHERE "{col}" IN {hex_subquery}'
|
||||||
|
)
|
||||||
|
for r in refs:
|
||||||
|
referenced.add(r[0])
|
||||||
|
print(f" {table:30s} {col:15s} → {len(refs)} hex refs")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
print(f" ERR {table}: {str(e)[:60]}")
|
||||||
|
|
||||||
|
orphaned = [c for c in hex_customers if c["name"] not in referenced]
|
||||||
|
still_used = [c for c in hex_customers if c["name"] in referenced]
|
||||||
|
|
||||||
|
print(f" Still referenced (keep): {len(still_used)}")
|
||||||
|
print(f" Orphaned (can delete): {len(orphaned)}")
|
||||||
|
|
||||||
|
if orphaned and not DRY_RUN:
|
||||||
|
# Delete orphaned hex customers using subquery (avoid ANY() limits)
|
||||||
|
# First, create a temp table of referenced hex customers
|
||||||
|
frappe.db.sql("""
|
||||||
|
DELETE FROM "tabDynamic Link"
|
||||||
|
WHERE link_doctype = 'Customer'
|
||||||
|
AND link_name IN (SELECT name FROM "tabCustomer"
|
||||||
|
WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
|
||||||
|
AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
deleted = frappe.db.sql("""
|
||||||
|
DELETE FROM "tabCustomer"
|
||||||
|
WHERE name ~ '^CUST-' AND name !~ '^CUST-[0-9]+$'
|
||||||
|
AND name NOT IN (SELECT DISTINCT party FROM "tabSubscription" WHERE party IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabSales Invoice" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT party FROM "tabPayment Entry" WHERE party IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabIssue" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Location" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Subscription" WHERE customer IS NOT NULL)
|
||||||
|
AND name NOT IN (SELECT DISTINCT customer FROM "tabService Equipment" WHERE customer IS NOT NULL)
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Deleted orphaned hex customers")
|
||||||
|
elif orphaned:
|
||||||
|
print(f" [DRY RUN] Would delete {len(orphaned)} orphaned hex customers")
|
||||||
|
else:
|
||||||
|
print(" No hex-pattern customers found.")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 6: VERIFY
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 6: VERIFICATION")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
new_sub_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription"')[0][0]
|
||||||
|
new_spd_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan Detail"')[0][0]
|
||||||
|
new_sp_count = frappe.db.sql('SELECT COUNT(*) FROM "tabSubscription Plan"')[0][0]
|
||||||
|
cust_count = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer"')[0][0]
|
||||||
|
|
||||||
|
print(f" Subscriptions: {new_sub_count}")
|
||||||
|
print(f" Plan Details: {new_spd_count}")
|
||||||
|
print(f" Plans: {new_sp_count}")
|
||||||
|
print(f" Customers: {cust_count}")
|
||||||
|
|
||||||
|
# Check for customer ID consistency: all subscriptions should point to sequential customers
|
||||||
|
hex_refs = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) FROM "tabSubscription"
|
||||||
|
WHERE party ~ '^CUST-' AND party !~ '^CUST-[0-9]+$'
|
||||||
|
""")[0][0]
|
||||||
|
print(f" Subscriptions pointing to hex customers: {hex_refs} (should be 0)")
|
||||||
|
|
||||||
|
# Spot-check: LPB4 customer (legacy_account_id = 4)
|
||||||
|
lpb4 = frappe.db.sql("""
|
||||||
|
SELECT name FROM "tabCustomer" WHERE legacy_account_id = 4
|
||||||
|
""", as_dict=True)
|
||||||
|
if lpb4:
|
||||||
|
cust_name = lpb4[0]["name"]
|
||||||
|
lpb4_subs = frappe.db.sql("""
|
||||||
|
SELECT actual_price, item_code, custom_description, status, billing_frequency
|
||||||
|
FROM "tabSubscription"
|
||||||
|
WHERE party = %s
|
||||||
|
ORDER BY actual_price DESC
|
||||||
|
""", (cust_name,), as_dict=True)
|
||||||
|
total_price = sum(float(s["actual_price"] or 0) for s in lpb4_subs if s["status"] == "Active")
|
||||||
|
print(f"\n Spot-check: {cust_name} (LPB4)")
|
||||||
|
for s in lpb4_subs:
|
||||||
|
print(f" {s['item_code'] or '':15s} {float(s['actual_price'] or 0):>8.2f} {s['billing_frequency']} {s['status']} {s['custom_description'] or ''}")
|
||||||
|
print(f" {'TOTAL':15s} {total_price:>8.2f}$/mo")
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("COMPLETE")
|
||||||
|
print("=" * 70)
|
||||||
|
print(f" Next: bench --site erp.gigafibre.ca clear-cache")
|
||||||
|
print()
|
||||||
581
scripts/migration/rename_all_doctypes.py
Normal file
581
scripts/migration/rename_all_doctypes.py
Normal file
|
|
@ -0,0 +1,581 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Rename all doctype IDs to zero-padded 10-digit numeric format.
|
||||||
|
|
||||||
|
Format: {PREFIX}-{legacy_id:010d}
|
||||||
|
SINV-0000001234 (Sales Invoice)
|
||||||
|
PE-0000001234 (Payment Entry)
|
||||||
|
ISS-0000001234 (Issue)
|
||||||
|
LOC-0000001234 (Service Location)
|
||||||
|
EQP-0000001234 (Service Equipment)
|
||||||
|
SUB-0000001234 (Service Subscription)
|
||||||
|
|
||||||
|
Two-phase rename to avoid PK collisions.
|
||||||
|
Uses temp mapping tables + bulk UPDATE ... FROM for FK references.
|
||||||
|
|
||||||
|
Run inside erpnext-backend-1:
|
||||||
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/rename_all_doctypes.py [--dry-run]
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
os.chdir("/home/frappe/frappe-bench/sites")
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
||||||
|
frappe.connect()
|
||||||
|
frappe.local.flags.ignore_permissions = True
|
||||||
|
print(f"Connected: {frappe.local.site}")
|
||||||
|
|
||||||
|
DRY_RUN = "--dry-run" in sys.argv
|
||||||
|
if DRY_RUN:
|
||||||
|
print("*** DRY RUN ***\n")
|
||||||
|
|
||||||
|
|
||||||
|
def rename_doctype(label, table, prefix, legacy_col, fk_refs, pad=10):
|
||||||
|
"""
|
||||||
|
Rename a doctype's records to {prefix}-{legacy_id:0{pad}d}.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
label: Human-readable label
|
||||||
|
table: PostgreSQL table name (e.g. "tabSales Invoice")
|
||||||
|
prefix: ID prefix (e.g. "SINV")
|
||||||
|
legacy_col: Column containing the legacy numeric ID
|
||||||
|
fk_refs: List of (fk_table, fk_column, extra_where) tuples
|
||||||
|
pad: Zero-padding width (default 10)
|
||||||
|
"""
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print(f"RENAMING: {label} → {prefix}-{('0' * pad)}")
|
||||||
|
print("=" * 70)
|
||||||
|
t_start = time.time()
|
||||||
|
|
||||||
|
# Step 1: Build mapping
|
||||||
|
rows = frappe.db.sql(f"""
|
||||||
|
SELECT name, {legacy_col} FROM "{table}"
|
||||||
|
WHERE {legacy_col} IS NOT NULL AND {legacy_col} > 0
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
mapping = {}
|
||||||
|
new_used = set()
|
||||||
|
collisions = 0
|
||||||
|
for r in rows:
|
||||||
|
old_name = r["name"]
|
||||||
|
new_name = f"{prefix}-{int(r[legacy_col]):0{pad}d}"
|
||||||
|
if new_name in new_used:
|
||||||
|
collisions += 1
|
||||||
|
new_name = f"{new_name}d{collisions}"
|
||||||
|
new_used.add(new_name)
|
||||||
|
if old_name != new_name:
|
||||||
|
mapping[old_name] = new_name
|
||||||
|
|
||||||
|
# Also handle records WITHOUT legacy ID
|
||||||
|
no_legacy = frappe.db.sql(f"""
|
||||||
|
SELECT COUNT(*) FROM "{table}"
|
||||||
|
WHERE {legacy_col} IS NULL OR {legacy_col} = 0
|
||||||
|
""")[0][0]
|
||||||
|
|
||||||
|
total = len(rows) + no_legacy
|
||||||
|
print(f" Total records: {total}")
|
||||||
|
print(f" Will rename: {len(mapping)}")
|
||||||
|
print(f" Already correct: {len(rows) - len(mapping)}")
|
||||||
|
print(f" No legacy ID (skip): {no_legacy}")
|
||||||
|
if collisions:
|
||||||
|
print(f" Collisions: {collisions}")
|
||||||
|
|
||||||
|
if not mapping:
|
||||||
|
print(" Nothing to rename!")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Show samples
|
||||||
|
samples = list(mapping.items())[:5]
|
||||||
|
for old, new in samples:
|
||||||
|
print(f" {old:30s} → {new}")
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Step 2: Create temp mapping table
|
||||||
|
map_table = f"_rename_map_{prefix.lower()}"
|
||||||
|
frappe.db.sql(f'DROP TABLE IF EXISTS {map_table}')
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
CREATE TEMP TABLE {map_table} (
|
||||||
|
old_name VARCHAR(140) PRIMARY KEY,
|
||||||
|
new_name VARCHAR(140) NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
# Insert mappings in batches
|
||||||
|
items = list(mapping.items())
|
||||||
|
batch_size = 1000
|
||||||
|
for start in range(0, len(items), batch_size):
|
||||||
|
batch = items[start:start + batch_size]
|
||||||
|
frappe.db.sql(
|
||||||
|
f"INSERT INTO {map_table} (old_name, new_name) VALUES " +
|
||||||
|
",".join(["(%s, %s)"] * len(batch)),
|
||||||
|
[v for pair in batch for v in pair]
|
||||||
|
)
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
# Step 3: Update FK references
|
||||||
|
for fk_table, fk_col, extra_where in fk_refs:
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "{fk_table}" t
|
||||||
|
SET "{fk_col}" = m.new_name
|
||||||
|
FROM {map_table} m
|
||||||
|
WHERE t."{fk_col}" = m.old_name
|
||||||
|
{extra_where}
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
err = str(e)[:80]
|
||||||
|
if "does not exist" not in err:
|
||||||
|
print(f" FK {fk_table}.{fk_col} ERR: {err}")
|
||||||
|
|
||||||
|
# Step 4: Two-phase rename
|
||||||
|
# Phase A: old → _TEMP_new
|
||||||
|
print(f" Phase A: temp rename...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '{old}' THEN '_TEMP_{new}'" for old, new in batch)
|
||||||
|
old_names = "','".join(old for old, _ in batch)
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "{table}"
|
||||||
|
SET name = CASE name {cases} END
|
||||||
|
WHERE name IN ('{old_names}')
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase A done [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
# Phase B: _TEMP_new → new
|
||||||
|
print(f" Phase B: final rename...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '_TEMP_{new}' THEN '{new}'" for _, new in batch)
|
||||||
|
temp_names = "','".join(f"_TEMP_{new}" for _, new in batch)
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "{table}"
|
||||||
|
SET name = CASE name {cases} END
|
||||||
|
WHERE name IN ('{temp_names}')
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase B done [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
frappe.db.sql(f'DROP TABLE IF EXISTS {map_table}')
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
elapsed = time.time() - t_start
|
||||||
|
print(f" ✓ {label} complete [{elapsed:.1f}s]")
|
||||||
|
|
||||||
|
|
||||||
|
def set_series(prefix, start_at):
|
||||||
|
"""Set the naming series counter."""
|
||||||
|
existing = frappe.db.sql(
|
||||||
|
'SELECT current FROM "tabSeries" WHERE name = %s', (prefix,), as_dict=True
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
if existing[0]["current"] < start_at:
|
||||||
|
frappe.db.sql('UPDATE "tabSeries" SET current = %s WHERE name = %s', (start_at, prefix))
|
||||||
|
print(f" Series {prefix} updated to {start_at}")
|
||||||
|
else:
|
||||||
|
print(f" Series {prefix} already at {existing[0]['current']}")
|
||||||
|
else:
|
||||||
|
frappe.db.sql('INSERT INTO "tabSeries" (name, current) VALUES (%s, %s)', (prefix, start_at))
|
||||||
|
print(f" Series {prefix} created at {start_at}")
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 1. SALES INVOICE
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# Current: SINV-1 through SINV-99999 (legacy invoice.id)
|
||||||
|
# The 'name' is already SINV-{id}, we just need to zero-pad
|
||||||
|
# Legacy column: the name itself encodes the ID. We need to extract it.
|
||||||
|
# Actually, let's check if there's a legacy column...
|
||||||
|
|
||||||
|
# Sales invoices were imported as SINV-{legacy_id}. The name IS the legacy ref.
|
||||||
|
# We need to extract the number from the name and zero-pad.
|
||||||
|
# Approach: use a direct SQL rename since legacy_id is embedded in name.
|
||||||
|
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("RENAMING: Sales Invoice → SINV-0000XXXXXX")
|
||||||
|
print("=" * 70)
|
||||||
|
t_start = time.time()
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
# Build mapping from current names
|
||||||
|
sinv_rows = frappe.db.sql("""
|
||||||
|
SELECT name FROM "tabSales Invoice" WHERE name ~ '^SINV-[0-9]+$'
|
||||||
|
""")
|
||||||
|
sinv_map = {}
|
||||||
|
for r in sinv_rows:
|
||||||
|
old = r[0]
|
||||||
|
num = int(old.replace("SINV-", ""))
|
||||||
|
new = f"SINV-{num:010d}"
|
||||||
|
if old != new:
|
||||||
|
sinv_map[old] = new
|
||||||
|
|
||||||
|
print(f" Will rename: {len(sinv_map)} invoices")
|
||||||
|
for old, new in list(sinv_map.items())[:5]:
|
||||||
|
print(f" {old:20s} → {new}")
|
||||||
|
|
||||||
|
if sinv_map:
|
||||||
|
# Create temp mapping
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _rename_sinv')
|
||||||
|
frappe.db.sql("""
|
||||||
|
CREATE TEMP TABLE _rename_sinv (
|
||||||
|
old_name VARCHAR(140) PRIMARY KEY,
|
||||||
|
new_name VARCHAR(140) NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
items = list(sinv_map.items())
|
||||||
|
for start in range(0, len(items), 1000):
|
||||||
|
batch = items[start:start + 1000]
|
||||||
|
frappe.db.sql(
|
||||||
|
"INSERT INTO _rename_sinv (old_name, new_name) VALUES " +
|
||||||
|
",".join(["(%s, %s)"] * len(batch)),
|
||||||
|
[v for pair in batch for v in pair]
|
||||||
|
)
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
# Update FK refs
|
||||||
|
sinv_fks = [
|
||||||
|
("tabSales Invoice Item", "parent", ""),
|
||||||
|
("tabSales Invoice Payment", "parent", ""),
|
||||||
|
("tabSales Taxes and Charges", "parent", "AND t.parenttype = 'Sales Invoice'"),
|
||||||
|
("tabGL Entry", "voucher_no", "AND t.voucher_type = 'Sales Invoice'"),
|
||||||
|
("tabPayment Ledger Entry", "voucher_no", "AND t.voucher_type = 'Sales Invoice'"),
|
||||||
|
("tabPayment Entry Reference", "reference_name", "AND t.reference_doctype = 'Sales Invoice'"),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Sales Invoice'"),
|
||||||
|
("tabVersion", "docname", "AND t.ref_doctype = 'Sales Invoice'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Sales Invoice'"),
|
||||||
|
]
|
||||||
|
for fk_table, fk_col, extra_where in sinv_fks:
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "{fk_table}" t
|
||||||
|
SET "{fk_col}" = m.new_name
|
||||||
|
FROM _rename_sinv m
|
||||||
|
WHERE t."{fk_col}" = m.old_name
|
||||||
|
{extra_where}
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
err = str(e)[:80]
|
||||||
|
if "does not exist" not in err:
|
||||||
|
print(f" FK {fk_table}.{fk_col} ERR: {err}")
|
||||||
|
|
||||||
|
# Two-phase rename
|
||||||
|
print(" Phase A...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '{old}' THEN '_T_{new}'" for old, new in batch)
|
||||||
|
old_names = "','".join(old for old, _ in batch)
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "tabSales Invoice"
|
||||||
|
SET name = CASE name {cases} END
|
||||||
|
WHERE name IN ('{old_names}')
|
||||||
|
""")
|
||||||
|
if (start + 500) % 10000 < 500:
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" {min(start + 500, len(items))}/{len(items)}")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase A done [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
print(" Phase B...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '_T_{new}' THEN '{new}'" for _, new in batch)
|
||||||
|
temp_names = "','".join(f"_T_{new}" for _, new in batch)
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "tabSales Invoice"
|
||||||
|
SET name = CASE name {cases} END
|
||||||
|
WHERE name IN ('{temp_names}')
|
||||||
|
""")
|
||||||
|
if (start + 500) % 10000 < 500:
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" {min(start + 500, len(items))}/{len(items)}")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase B done [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _rename_sinv')
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
print(f" ✓ Sales Invoice [{time.time()-t_start:.1f}s]")
|
||||||
|
else:
|
||||||
|
sinv_count = frappe.db.sql("SELECT COUNT(*) FROM \"tabSales Invoice\" WHERE name ~ '^SINV-[0-9]+$'")[0][0]
|
||||||
|
print(f" [DRY] Would rename ~{sinv_count} invoices")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 2. PAYMENT ENTRY — same pattern as SINV
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("RENAMING: Payment Entry → PE-0000XXXXXX")
|
||||||
|
print("=" * 70)
|
||||||
|
t_start = time.time()
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
pe_rows = frappe.db.sql("SELECT name FROM \"tabPayment Entry\" WHERE name ~ '^PE-[0-9]+$'")
|
||||||
|
pe_map = {}
|
||||||
|
for r in pe_rows:
|
||||||
|
old = r[0]
|
||||||
|
num = int(old.replace("PE-", ""))
|
||||||
|
new = f"PE-{num:010d}"
|
||||||
|
if old != new:
|
||||||
|
pe_map[old] = new
|
||||||
|
|
||||||
|
print(f" Will rename: {len(pe_map)} payments")
|
||||||
|
for old, new in list(pe_map.items())[:3]:
|
||||||
|
print(f" {old:20s} → {new}")
|
||||||
|
|
||||||
|
if pe_map:
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _rename_pe')
|
||||||
|
frappe.db.sql("""
|
||||||
|
CREATE TEMP TABLE _rename_pe (
|
||||||
|
old_name VARCHAR(140) PRIMARY KEY, new_name VARCHAR(140) NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
items = list(pe_map.items())
|
||||||
|
for start in range(0, len(items), 1000):
|
||||||
|
batch = items[start:start + 1000]
|
||||||
|
frappe.db.sql(
|
||||||
|
"INSERT INTO _rename_pe (old_name, new_name) VALUES " +
|
||||||
|
",".join(["(%s, %s)"] * len(batch)),
|
||||||
|
[v for pair in batch for v in pair]
|
||||||
|
)
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
pe_fks = [
|
||||||
|
("tabPayment Entry Reference", "parent", ""),
|
||||||
|
("tabGL Entry", "voucher_no", "AND t.voucher_type = 'Payment Entry'"),
|
||||||
|
("tabPayment Ledger Entry", "voucher_no", "AND t.voucher_type = 'Payment Entry'"),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Payment Entry'"),
|
||||||
|
("tabVersion", "docname", "AND t.ref_doctype = 'Payment Entry'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Payment Entry'"),
|
||||||
|
]
|
||||||
|
for fk_table, fk_col, extra_where in pe_fks:
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
frappe.db.sql(f"""
|
||||||
|
UPDATE "{fk_table}" t SET "{fk_col}" = m.new_name
|
||||||
|
FROM _rename_pe m WHERE t."{fk_col}" = m.old_name {extra_where}
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" FK {fk_table}.{fk_col} [{time.time()-t0:.1f}s]")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
err = str(e)[:80]
|
||||||
|
if "does not exist" not in err:
|
||||||
|
print(f" FK {fk_table}.{fk_col} ERR: {err}")
|
||||||
|
|
||||||
|
# Two-phase rename
|
||||||
|
print(" Phase A...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '{old}' THEN '_T_{new}'" for old, new in batch)
|
||||||
|
old_names = "','".join(old for old, _ in batch)
|
||||||
|
frappe.db.sql(f'UPDATE "tabPayment Entry" SET name = CASE name {cases} END WHERE name IN (\'{old_names}\')')
|
||||||
|
if (start + 500) % 10000 < 500:
|
||||||
|
frappe.db.commit()
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase A [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
print(" Phase B...")
|
||||||
|
t0 = time.time()
|
||||||
|
for start in range(0, len(items), 500):
|
||||||
|
batch = items[start:start + 500]
|
||||||
|
cases = " ".join(f"WHEN '_T_{new}' THEN '{new}'" for _, new in batch)
|
||||||
|
temp_names = "','".join(f"_T_{new}" for _, new in batch)
|
||||||
|
frappe.db.sql(f'UPDATE "tabPayment Entry" SET name = CASE name {cases} END WHERE name IN (\'{temp_names}\')')
|
||||||
|
if (start + 500) % 10000 < 500:
|
||||||
|
frappe.db.commit()
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Phase B [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _rename_pe')
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
print(f" ✓ Payment Entry [{time.time()-t_start:.1f}s]")
|
||||||
|
else:
|
||||||
|
pe_count = frappe.db.sql("SELECT COUNT(*) FROM \"tabPayment Entry\" WHERE name ~ '^PE-[0-9]+$'")[0][0]
|
||||||
|
print(f" [DRY] Would rename ~{pe_count} payments")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 3. ISSUE — has legacy_ticket_id column
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
rename_doctype(
|
||||||
|
label="Issue",
|
||||||
|
table="tabIssue",
|
||||||
|
prefix="ISS",
|
||||||
|
legacy_col="legacy_ticket_id",
|
||||||
|
fk_refs=[
|
||||||
|
("tabDispatch Job", "issue", ""),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Issue'"),
|
||||||
|
("tabVersion", "docname", "AND t.ref_doctype = 'Issue'"),
|
||||||
|
("tabCommunication", "reference_name", "AND t.reference_doctype = 'Issue'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Issue'"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 4. SERVICE LOCATION — has legacy_delivery_id column
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
rename_doctype(
|
||||||
|
label="Service Location",
|
||||||
|
table="tabService Location",
|
||||||
|
prefix="LOC",
|
||||||
|
legacy_col="legacy_delivery_id",
|
||||||
|
fk_refs=[
|
||||||
|
("tabService Subscription", "service_location", ""),
|
||||||
|
("tabSubscription", "service_location", ""),
|
||||||
|
("tabService Equipment", "service_location", ""),
|
||||||
|
("tabIssue", "service_location", ""),
|
||||||
|
("tabDispatch Job", "service_location", ""),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Service Location'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Location'"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 5. SERVICE EQUIPMENT — has legacy_device_id column
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
rename_doctype(
|
||||||
|
label="Service Equipment",
|
||||||
|
table="tabService Equipment",
|
||||||
|
prefix="EQP",
|
||||||
|
legacy_col="legacy_device_id",
|
||||||
|
fk_refs=[
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Service Equipment'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Equipment'"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 6. SERVICE SUBSCRIPTION — has legacy_service_id column
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
rename_doctype(
|
||||||
|
label="Service Subscription",
|
||||||
|
table="tabService Subscription",
|
||||||
|
prefix="SUB",
|
||||||
|
legacy_col="legacy_service_id",
|
||||||
|
fk_refs=[
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Service Subscription'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Service Subscription'"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# 7. SUBSCRIPTION (native) — also has legacy_service_id
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
rename_doctype(
|
||||||
|
label="Subscription (native)",
|
||||||
|
table="tabSubscription",
|
||||||
|
prefix="ASUB",
|
||||||
|
legacy_col="legacy_service_id",
|
||||||
|
fk_refs=[
|
||||||
|
("tabSubscription Plan Detail", "parent", ""),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Subscription'"),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Subscription'"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# SET NAMING SERIES COUNTERS
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("NAMING SERIES COUNTERS")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
if not DRY_RUN:
|
||||||
|
set_series("SINV-", 100000)
|
||||||
|
set_series("PE-", 100000)
|
||||||
|
set_series("ISS-", 250000) # max legacy=244240
|
||||||
|
set_series("LOC-", 100000)
|
||||||
|
set_series("EQP-", 100000)
|
||||||
|
set_series("SUB-", 100000) # max legacy=74184
|
||||||
|
set_series("ASUB-", 100000)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# VERIFICATION
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("VERIFICATION")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
checks = [
|
||||||
|
("tabSales Invoice", "SINV-"),
|
||||||
|
("tabPayment Entry", "PE-"),
|
||||||
|
("tabIssue", "ISS-"),
|
||||||
|
("tabService Location", "LOC-"),
|
||||||
|
("tabService Equipment", "EQP-"),
|
||||||
|
("tabService Subscription", "SUB-"),
|
||||||
|
("tabSubscription", "ASUB-"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for table, prefix in checks:
|
||||||
|
try:
|
||||||
|
total = frappe.db.sql(f'SELECT COUNT(*) FROM "{table}"')[0][0]
|
||||||
|
correct = frappe.db.sql(f"""
|
||||||
|
SELECT COUNT(*) FROM "{table}"
|
||||||
|
WHERE name ~ %s
|
||||||
|
""", (f'^{prefix}[0-9]{{10}}$',))[0][0]
|
||||||
|
print(f" {table:30s} total={total:>8d} correct_format={correct:>8d} {'✓' if total == correct else f'({total - correct} remaining)'}")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
print(f" {table}: ERR {str(e)[:50]}")
|
||||||
|
|
||||||
|
# Spot checks
|
||||||
|
print("\n Spot checks:")
|
||||||
|
for table, field, value in [
|
||||||
|
("tabSales Invoice", "name", None),
|
||||||
|
("tabPayment Entry", "name", None),
|
||||||
|
("tabIssue", "name", None),
|
||||||
|
]:
|
||||||
|
sample = frappe.db.sql(f'SELECT name FROM "{table}" ORDER BY name LIMIT 3')
|
||||||
|
print(f" {table}: {[s[0] for s in sample]}")
|
||||||
|
|
||||||
|
# FK orphan checks
|
||||||
|
print("\n FK consistency:")
|
||||||
|
fk_checks = [
|
||||||
|
("tabSales Invoice Item", "parent", "tabSales Invoice"),
|
||||||
|
("tabPayment Entry Reference", "parent", "tabPayment Entry"),
|
||||||
|
("tabGL Entry", "voucher_no", None), # multi-type, skip
|
||||||
|
("tabSubscription Plan Detail", "parent", "tabSubscription"),
|
||||||
|
]
|
||||||
|
for fk_table, fk_col, parent_table in fk_checks:
|
||||||
|
if not parent_table:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
orphans = frappe.db.sql(f"""
|
||||||
|
SELECT COUNT(*) FROM "{fk_table}" t
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM "{parent_table}" p WHERE p.name = t."{fk_col}")
|
||||||
|
""")[0][0]
|
||||||
|
status = "✓" if orphans == 0 else f"WARNING: {orphans} orphaned!"
|
||||||
|
print(f" {fk_table}.{fk_col} → {parent_table}: {status}")
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
|
||||||
|
frappe.clear_cache()
|
||||||
|
print("\n✓ All done — cache cleared")
|
||||||
|
print(" Next: bench --site erp.gigafibre.ca clear-cache")
|
||||||
329
scripts/migration/rename_customers.py
Normal file
329
scripts/migration/rename_customers.py
Normal file
|
|
@ -0,0 +1,329 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Rename all Customer IDs to CUST-{legacy_customer_id}.
|
||||||
|
|
||||||
|
Handles all FK references across the entire ERPNext database:
|
||||||
|
- Sales Invoice, Payment Entry, GL Entry, PLE, Issues, Locations,
|
||||||
|
Subscriptions, Equipment, Dispatch Jobs, Dynamic Links, Comments, etc.
|
||||||
|
|
||||||
|
Approach:
|
||||||
|
1. Build old_name → new_name mapping from legacy_customer_id
|
||||||
|
2. Create temp mapping table in PostgreSQL
|
||||||
|
3. Bulk UPDATE each FK table using JOIN to mapping table
|
||||||
|
4. Rename Customer records themselves
|
||||||
|
5. Set naming series counter for new customers
|
||||||
|
6. Verify all references updated
|
||||||
|
|
||||||
|
Run inside erpnext-backend-1:
|
||||||
|
/home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/rename_customers.py [--dry-run]
|
||||||
|
|
||||||
|
Estimated time: 5-10 minutes (millions of rows to update)
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
os.chdir("/home/frappe/frappe-bench/sites")
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
frappe.init(site="erp.gigafibre.ca", sites_path=".")
|
||||||
|
frappe.connect()
|
||||||
|
frappe.local.flags.ignore_permissions = True
|
||||||
|
print(f"Connected: {frappe.local.site}")
|
||||||
|
|
||||||
|
DRY_RUN = "--dry-run" in sys.argv
|
||||||
|
# Use legacy customer_id as the new name: CUST-{legacy_customer_id}
|
||||||
|
USE_LEGACY_CODE = "--legacy-code" in sys.argv
|
||||||
|
# Use zero-padded numeric: CUST-{legacy_account_id:015d}
|
||||||
|
USE_NUMERIC = "--numeric" in sys.argv
|
||||||
|
|
||||||
|
if not USE_LEGACY_CODE and not USE_NUMERIC:
|
||||||
|
USE_LEGACY_CODE = True # default
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
print("*** DRY RUN — no changes will be written ***")
|
||||||
|
|
||||||
|
mode_label = "legacy_customer_id" if USE_LEGACY_CODE else "zero-padded numeric"
|
||||||
|
print(f"Naming mode: {mode_label}")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 1: BUILD MAPPING
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 1: BUILD NAME MAPPING")
|
||||||
|
print("=" * 70)
|
||||||
|
t0 = time.time()
|
||||||
|
|
||||||
|
customers = frappe.db.sql("""
|
||||||
|
SELECT name, legacy_account_id, legacy_customer_id, customer_name
|
||||||
|
FROM "tabCustomer"
|
||||||
|
ORDER BY legacy_account_id
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
mapping = {} # old_name → new_name
|
||||||
|
collisions = []
|
||||||
|
new_names_used = set()
|
||||||
|
|
||||||
|
for c in customers:
|
||||||
|
old_name = c["name"]
|
||||||
|
|
||||||
|
if USE_LEGACY_CODE:
|
||||||
|
cid = c.get("legacy_customer_id") or ""
|
||||||
|
if cid:
|
||||||
|
new_name = f"CUST-{cid}"
|
||||||
|
else:
|
||||||
|
# Fallback to zero-padded account ID
|
||||||
|
new_name = f"CUST-{c['legacy_account_id']:015d}"
|
||||||
|
else:
|
||||||
|
new_name = f"CUST-{c['legacy_account_id']:015d}"
|
||||||
|
|
||||||
|
# Check for collision
|
||||||
|
if new_name in new_names_used:
|
||||||
|
collisions.append((old_name, new_name, c["customer_name"]))
|
||||||
|
# Append account ID to disambiguate
|
||||||
|
new_name = f"{new_name}-{c['legacy_account_id']}"
|
||||||
|
|
||||||
|
new_names_used.add(new_name)
|
||||||
|
|
||||||
|
if old_name != new_name:
|
||||||
|
mapping[old_name] = new_name
|
||||||
|
|
||||||
|
# Stats
|
||||||
|
unchanged = len(customers) - len(mapping)
|
||||||
|
print(f" Total customers: {len(customers)}")
|
||||||
|
print(f" Will rename: {len(mapping)}")
|
||||||
|
print(f" Already correct: {unchanged}")
|
||||||
|
print(f" Collisions resolved: {len(collisions)}")
|
||||||
|
|
||||||
|
if collisions:
|
||||||
|
print(f" Collision examples:")
|
||||||
|
for old, new, name in collisions[:5]:
|
||||||
|
print(f" {old} → {new} ({name})")
|
||||||
|
|
||||||
|
# Show sample mappings
|
||||||
|
print(f"\n Sample renames:")
|
||||||
|
for old, new in list(mapping.items())[:10]:
|
||||||
|
cust = next((c for c in customers if c["name"] == old), {})
|
||||||
|
print(f" {old:25s} → {new:35s} ({cust.get('customer_name', '')})")
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
print(f"\n [{time.time()-t0:.1f}s]")
|
||||||
|
print("\n*** DRY RUN complete — no changes made ***")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 2: CREATE TEMP MAPPING TABLE
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 2: CREATE TEMP MAPPING TABLE")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _customer_rename_map')
|
||||||
|
frappe.db.sql("""
|
||||||
|
CREATE TEMP TABLE _customer_rename_map (
|
||||||
|
old_name VARCHAR(140) PRIMARY KEY,
|
||||||
|
new_name VARCHAR(140) NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
# Insert in batches
|
||||||
|
batch = []
|
||||||
|
for old, new in mapping.items():
|
||||||
|
batch.append((old, new))
|
||||||
|
if len(batch) >= 1000:
|
||||||
|
frappe.db.sql(
|
||||||
|
"INSERT INTO _customer_rename_map (old_name, new_name) VALUES " +
|
||||||
|
",".join(["(%s, %s)"] * len(batch)),
|
||||||
|
[v for pair in batch for v in pair]
|
||||||
|
)
|
||||||
|
batch = []
|
||||||
|
if batch:
|
||||||
|
frappe.db.sql(
|
||||||
|
"INSERT INTO _customer_rename_map (old_name, new_name) VALUES " +
|
||||||
|
",".join(["(%s, %s)"] * len(batch)),
|
||||||
|
[v for pair in batch for v in pair]
|
||||||
|
)
|
||||||
|
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Inserted {len(mapping)} mappings into temp table")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 3: UPDATE ALL FK REFERENCES
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 3: UPDATE FK REFERENCES")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
# All tables with customer references: (table, column, extra_where)
|
||||||
|
fk_tables = [
|
||||||
|
("tabSales Invoice", "customer", ""),
|
||||||
|
("tabPayment Entry", "party", ""),
|
||||||
|
("tabGL Entry", "party", ""),
|
||||||
|
("tabPayment Ledger Entry", "party", ""),
|
||||||
|
("tabIssue", "customer", ""),
|
||||||
|
("tabService Location", "customer", ""),
|
||||||
|
("tabService Subscription", "customer", ""),
|
||||||
|
("tabSubscription", "party", ""),
|
||||||
|
("tabService Equipment", "customer", ""),
|
||||||
|
("tabDispatch Job", "customer", ""),
|
||||||
|
("tabDynamic Link", "link_name", "AND t.link_doctype = 'Customer'"),
|
||||||
|
("tabComment", "reference_name", "AND t.reference_doctype = 'Customer'"),
|
||||||
|
("tabCommunication", "reference_name", "AND t.reference_doctype = 'Customer'"),
|
||||||
|
# Version log
|
||||||
|
("tabVersion", "docname", "AND t.ref_doctype = 'Customer'"),
|
||||||
|
# Sales taxes (parent references invoice, not customer directly)
|
||||||
|
# Payment Entry Reference has no direct customer column
|
||||||
|
]
|
||||||
|
|
||||||
|
total_updated = 0
|
||||||
|
for table, col, extra_where in fk_tables:
|
||||||
|
t0 = time.time()
|
||||||
|
try:
|
||||||
|
result = frappe.db.sql(f"""
|
||||||
|
UPDATE "{table}" t
|
||||||
|
SET "{col}" = m.new_name
|
||||||
|
FROM _customer_rename_map m
|
||||||
|
WHERE t."{col}" = m.old_name
|
||||||
|
{extra_where}
|
||||||
|
""")
|
||||||
|
# Get affected rows
|
||||||
|
affected = frappe.db.sql(f"""
|
||||||
|
SELECT COUNT(*) FROM "{table}" t
|
||||||
|
INNER JOIN _customer_rename_map m ON t."{col}" = m.old_name
|
||||||
|
{extra_where.replace('t.', f'"{table}".')}
|
||||||
|
""")
|
||||||
|
# Actually just count what we updated - the UPDATE already ran
|
||||||
|
frappe.db.commit()
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
print(f" {table:35s} {col:20s} updated [{elapsed:.1f}s]")
|
||||||
|
total_updated += 1
|
||||||
|
except Exception as e:
|
||||||
|
frappe.db.rollback()
|
||||||
|
err_msg = str(e)[:80]
|
||||||
|
if "does not exist" in err_msg or "column" in err_msg:
|
||||||
|
pass # Table/column doesn't exist, skip
|
||||||
|
else:
|
||||||
|
print(f" {table:35s} {col:20s} ERR: {err_msg}")
|
||||||
|
|
||||||
|
print(f"\n Updated {total_updated} FK tables")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 4: RENAME CUSTOMER RECORDS THEMSELVES
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 4: RENAME CUSTOMER RECORDS")
|
||||||
|
print("=" * 70)
|
||||||
|
t0 = time.time()
|
||||||
|
|
||||||
|
# Must rename customer records — update the `name` column
|
||||||
|
# Do this AFTER FK updates to avoid FK constraint issues
|
||||||
|
frappe.db.sql("""
|
||||||
|
UPDATE "tabCustomer" t
|
||||||
|
SET name = m.new_name
|
||||||
|
FROM _customer_rename_map m
|
||||||
|
WHERE t.name = m.old_name
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(f" Renamed {len(mapping)} customers [{time.time()-t0:.1f}s]")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 5: SET NAMING SERIES COUNTER
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 5: SET NAMING SERIES COUNTER")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
# For new customers, ERPNext will use naming_series.
|
||||||
|
# We need to make sure it doesn't collide.
|
||||||
|
# Since legacy customer_ids are alphanumeric, numeric auto-increment won't collide.
|
||||||
|
# But let's set a safe starting point anyway.
|
||||||
|
|
||||||
|
# Check if CUST- series exists
|
||||||
|
series = frappe.db.sql("""
|
||||||
|
SELECT name, current FROM "tabSeries" WHERE name = 'CUST-'
|
||||||
|
""", as_dict=True)
|
||||||
|
|
||||||
|
if series:
|
||||||
|
print(f" Current CUST- series: {series[0]['current']}")
|
||||||
|
else:
|
||||||
|
print(" No CUST- series found, creating...")
|
||||||
|
frappe.db.sql("""
|
||||||
|
INSERT INTO "tabSeries" (name, current) VALUES ('CUST-', 100000)
|
||||||
|
ON CONFLICT (name) DO UPDATE SET current = GREATEST("tabSeries".current, 100000)
|
||||||
|
""")
|
||||||
|
frappe.db.commit()
|
||||||
|
print(" Set CUST- counter to 100000")
|
||||||
|
|
||||||
|
# Note: The naming_series format should produce CUST-NNNNN or CUST-.#####
|
||||||
|
# New customers will get CUST-100001, CUST-100002, etc.
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 6: CLEANUP TEMP TABLE
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
frappe.db.sql('DROP TABLE IF EXISTS _customer_rename_map')
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# STEP 7: VERIFY
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("STEP 7: VERIFICATION")
|
||||||
|
print("=" * 70)
|
||||||
|
|
||||||
|
cust_count = frappe.db.sql('SELECT COUNT(*) FROM "tabCustomer"')[0][0]
|
||||||
|
print(f" Customers: {cust_count}")
|
||||||
|
|
||||||
|
# Check for any remaining hex-pattern IDs
|
||||||
|
hex_remaining = frappe.db.sql("""
|
||||||
|
SELECT COUNT(*) FROM "tabCustomer"
|
||||||
|
WHERE name ~ '^CUST-[0-9a-f]{8,}'
|
||||||
|
AND name !~ '^CUST-[0-9]+$'
|
||||||
|
""")[0][0]
|
||||||
|
print(f" Hex-pattern IDs remaining: {hex_remaining} (should be 0)")
|
||||||
|
|
||||||
|
# Spot checks
|
||||||
|
spots = [
|
||||||
|
(4, "LPB4"),
|
||||||
|
(2393, "Vegpro"),
|
||||||
|
(13814, "Vegpro 114796350603272"),
|
||||||
|
]
|
||||||
|
for legacy_id, label in spots:
|
||||||
|
c = frappe.db.sql("""
|
||||||
|
SELECT name, customer_name, legacy_customer_id
|
||||||
|
FROM "tabCustomer" WHERE legacy_account_id = %s
|
||||||
|
""", (legacy_id,), as_dict=True)
|
||||||
|
if c:
|
||||||
|
print(f" {label}: {c[0]['name']} ({c[0]['customer_name']})")
|
||||||
|
|
||||||
|
# Check FK consistency: any orphaned references?
|
||||||
|
for table, col in [("tabSales Invoice", "customer"), ("tabSubscription", "party"), ("tabIssue", "customer")]:
|
||||||
|
orphans = frappe.db.sql(f"""
|
||||||
|
SELECT COUNT(*) FROM "{table}" t
|
||||||
|
WHERE t."{col}" IS NOT NULL
|
||||||
|
AND t."{col}" != ''
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM "tabCustomer" c WHERE c.name = t."{col}")
|
||||||
|
""")[0][0]
|
||||||
|
if orphans > 0:
|
||||||
|
print(f" WARNING: {orphans} orphaned references in {table}.{col}!")
|
||||||
|
else:
|
||||||
|
print(f" {table}.{col}: all references valid ✓")
|
||||||
|
|
||||||
|
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
# DONE
|
||||||
|
# ═══════════════════════════════════════════════════════════════
|
||||||
|
print("\n" + "=" * 70)
|
||||||
|
print("COMPLETE")
|
||||||
|
print("=" * 70)
|
||||||
|
print(" Next: bench --site erp.gigafibre.ca clear-cache")
|
||||||
|
|
||||||
|
frappe.clear_cache()
|
||||||
|
print(" Cache cleared")
|
||||||
Loading…
Reference in New Issue
Block a user