diff --git a/apps/ops/src/pages/LegacySyncPage.vue b/apps/ops/src/pages/LegacySyncPage.vue index 142bfc8..d8d6204 100644 --- a/apps/ops/src/pages/LegacySyncPage.vue +++ b/apps/ops/src/pages/LegacySyncPage.vue @@ -32,6 +32,51 @@ + + + + +
+
Réconciliation par module — F vs ERPNext
+
Comptes bruts de chaque côté. Écart = F − ERPNext (positif = manque dans ERPNext).
+
+ + Recalculer les écarts + + + + + Module + F (legacy) + ERPNext + Écart + Note + + + + + {{ e.label }} + {{ fmtN(e.f) }} + {{ fmtN(e.erp) }} + + {{ e.delta == null ? 'n/a' : (e.delta > 0 ? '+' : '') + fmtN(e.delta) }} + + {{ e.note }} + + + + + {{ fmtN(scoreboard.resiliated_f_accounts) }} comptes F résiliés + · + + {{ fmtN(scoreboard.ghost_active_subscriptions) }} abonnements fantômes (compte résilié) + + + {{ new Date(scoreboard.generated_at).toLocaleString('fr-CA') }} + + Calcul des écarts par module… + + @@ -197,6 +242,9 @@ const sd = computed(() => (rep.value && rep.value.customers.status_divergence) | // ── Bande unifiée (rapide, /sync/status) ── const sLoading = ref(false) const status = ref(null) +// ── Tableau de bord de réconciliation (comptes bruts F vs ERPNext, /legacy-sync/scoreboard) ── +const scoreboard = ref(null) +const sbLoading = ref(false) const health = computed(() => (status.value && status.value.billing_health) || {}) const fmtN = (v) => v == null ? '—' : Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 }) const fmtCur = (v) => v == null ? '—' : '$' + Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 }) @@ -230,7 +278,12 @@ async function load () { } catch (e) { err.value = e.message } loading.value = false } -function reload () { loadStatus(); load() } +async function loadScoreboard () { + sbLoading.value = true + try { const r = await fetch(`${HUB_URL}/legacy-sync/scoreboard`); if (r.ok) scoreboard.value = await r.json() } catch {} finally { sbLoading.value = false } +} +function deltaClass (d) { if (d == null) return 'text-grey-5'; if (d === 0) return 'text-green-7'; if (Math.abs(d) < 100) return 'text-orange-8'; return 'text-red-7' } +function reload () { loadStatus(); load(); loadScoreboard() } // ── Synchronisation manuelle + progression ── const syncing = ref(false) diff --git a/scripts/targo-sync/README.md b/scripts/targo-sync/README.md new file mode 100644 index 0000000..77d8b15 --- /dev/null +++ b/scripts/targo-sync/README.md @@ -0,0 +1,59 @@ +# targo-sync — miroir opérationnel F (legacy) → ERPNext + +Snapshot versionné des scripts qui tournent sur **l'hôte de prod** dans `/opt/targo-sync/` +(hors conteneur). Avant 2026-07-06 ils n'étaient dans aucun repo — c'est le vrai moteur de +synchronisation « legacy F → ERPNext » pour la couche **opérationnelle** (ce que les équipes +voient : clients, adresses, services/abonnements, factures, paiements). **F reste autoritaire** +pour la facturation (scheduler ERPNext en pause) ; ERPNext = miroir + grand livre à la bascule. + +## Cadence + +Cron horaire sur l'hôte (`15 * * * *`) → `run.sh` → 4 étapes, dans l'ordre du DAG de dépendances : + +1. **Comptes manquants** — hub `POST /legacy-payments/ensure-customers` (crée les Customer absents). +2. **Adresses + services** — `sync_services_incremental.py` (Python, exécuté **dans** `erpnext-backend-1`). +3. **Factures** — `sync_invoices_incremental.py` (idem). +4. **Paiements + soldes** — hub `POST /legacy-payments/sync-cycle`. + +Les scripts Python sont idempotents (`ON CONFLICT DO NOTHING` pour les créations, `UPDATE` +conditionnel pour les rafraîchissements). **`APPLY=0` = dry-run** (n'écrit rien) ; `run.sh` lance +en `APPLY=1`. Autres env : `PG_HOST` (défaut `db`), `LIMIT` (taille de lot), `LEGACY_HOST`. + +## Déploiement (host, hors repo) + +```sh +scp scripts/targo-sync/*.py scripts/targo-sync/run.sh root@:/opt/targo-sync/ +# le cron docker cp le .py dans erpnext-backend-1 puis l'exécute — pas de restart requis +``` + +Dry-run manuel d'un script avant bascule : + +```sh +docker cp /opt/targo-sync/sync_services_incremental.py erpnext-backend-1:/tmp/svc.py +docker exec -e APPLY=0 -e PG_HOST=db erpnext-backend-1 \ + /home/frappe/frappe-bench/env/bin/python /tmp/svc.py +``` + +## ⚠️ FIX 2026-07-06 — abonnements fantômes (compte résilié affiché « Actif ») + +En F, résilier un **compte** met `account.status ∈ {3,4,5}` mais **laisse les services à +`service.status=1`** (F ne cascade pas). `svc_status()` dérivait le statut du seul +`service.status` → un compte résilié ressortait « Actif » dans ERPNext. Comme le statut client +est dérivé des abonnements, ~2 663 ex-clients apparaissaient actifs et ~3 855 abonnements +étaient des fantômes (MRR gonflé, audiences polluées). + +Correctif dans `sync_services_incremental.py` (marqueur `FIX 2026-07-06`) : `svc_status()` charge +une fois `SELECT id FROM account WHERE status IN (3,4,5)` et **force `Annulé` dès que le compte est +résilié**, sur les deux chemins (Phase B création + Phase D rafraîchissement). Sans ce garde-fou, +le cron ré-activait les fantômes à chaque heure. Voir la mémoire projet `feedback_ghost_active_subscriptions`. + +Réconciliation visible dans OPS : page **/sync-legacy** → carte « Réconciliation par module » +(endpoint hub `GET /legacy-sync/scoreboard`, métrique `ghost_active_subscriptions` = doit rester 0). + +## Fichiers + +| Fichier | Rôle | +|---|---| +| `run.sh` | Orchestrateur cron (4 étapes DAG) | +| `sync_services_incremental.py` | Service Location (delivery) + Service Subscription (service) : création + prix + **statut (fix fantômes)** | +| `sync_invoices_incremental.py` | Sales Invoice (miroir des factures F ; statut reflétant F) | diff --git a/scripts/targo-sync/run.sh b/scripts/targo-sync/run.sh new file mode 100755 index 0000000..82d5aa3 --- /dev/null +++ b/scripts/targo-sync/run.sh @@ -0,0 +1,26 @@ +#!/bin/bash +LOG=/opt/targo-sync/billing-sync.log; [ -f "$LOG" ] && tail -n 1500 "$LOG" > "$LOG.tmp" 2>/dev/null && mv "$LOG.tmp" "$LOG" +# Sync F→ERPNext (miroir opérationnel) — idempotent, léger, gap-aware. Ordre = DAG des dépendances. +set -uo pipefail +LOG=/opt/targo-sync/billing-sync.log +exec >> "$LOG" 2>&1 +echo "===== $(date -Is) =====" +echo "-- 1. comptes manquants (hub) --" +docker exec targo-hub node -e ' + const T=process.env.HUB_SERVICE_TOKEN + fetch("http://localhost:3300/legacy-payments/ensure-customers",{method:"POST",headers:{Authorization:"Bearer "+T,"Content-Type":"application/json"},body:JSON.stringify({confirm:"F-WINS",limit:1000})}) + .then(r=>r.json()).then(d=>console.log("comptes:",JSON.stringify({missing:d.missing,created:d.created,errors:d.errors}))).catch(e=>console.log("comptes ERR",e.message)) +' +echo "-- 2. adresses + services (Python, opérationnel) --" +docker cp /opt/targo-sync/sync_services_incremental.py erpnext-backend-1:/tmp/sync_svc.py 2>/dev/null +docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py 2>&1 | tail -4 +echo "-- 3. factures (Python, statut reflétant F) --" +docker cp /opt/targo-sync/sync_invoices_incremental.py erpnext-backend-1:/tmp/sync_inv.py 2>/dev/null +docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py 2>&1 | tail -2 +echo "-- 4. paiements + soldes (hub) --" +docker exec targo-hub node -e ' + const T=process.env.HUB_SERVICE_TOKEN + fetch("http://localhost:3300/legacy-payments/sync-cycle",{method:"POST",headers:{Authorization:"Bearer "+T,"Content-Type":"application/json"},body:JSON.stringify({confirm:"F-WINS",limit:20000})}) + .then(r=>r.json()).then(d=>console.log("cycle:",JSON.stringify({pay:d.payments&&d.payments.applied,refreshed:d.open_refresh&&d.open_refresh.updated}))).catch(e=>{console.log("cycle ERR",e.message)}) +' +echo "done $(date -Is)" diff --git a/scripts/targo-sync/sync_invoices_incremental.py b/scripts/targo-sync/sync_invoices_incremental.py new file mode 100644 index 0000000..8111896 --- /dev/null +++ b/scripts/targo-sync/sync_invoices_incremental.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +Sync INCRÉMENTAL des factures F (gestionclient) → ERPNext — MIROIR OPÉRATIONNEL. + +Insère les factures F absentes d'ERPNext (id > watermark) en REFLÉTANT leur statut F : + • Sales Invoice + Sales Invoice Item (mêmes maps que import_invoices.py → cohérence + exacte avec les ~629k déjà importées : client, SKU→compte produit, service_location). + • docstatus = 1 (submitted), status = Paid/Unpaid/Overdue/Return d'après F, + outstanding_amount = max(0, total_amt − billed_amt) (F = vérité du solde). + • SINV-{legacy_invoice_id zfill10} (idempotent : saute si déjà présent). + • PAS de GL Entry / PLE (ERPNext non autoritaire — miroir opérationnel, scheduler en pause). + +LÉGER SUR F : ne lit que `WHERE id > watermark` (PK indexée) sur F LIVE. Aucun scan complet. +DRY-RUN par défaut : n'écrit RIEN sans la variable d'env APPLY=1. + +Lancer (depuis l'hôte prod qui joint F live 10.100.80.100 + le PG ERPNext) : + docker cp sync_invoices_incremental.py erpnext-backend-1:/tmp/sync_inv.py + docker exec -e APPLY=0 erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py # aperçu + docker exec -e APPLY=1 erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py # écrit +Options env : SINCE (force le plancher d'id), LIMIT (taille du lot, 0=tout). +""" +import os +import uuid +import pymysql +import psycopg2 +from datetime import datetime, timezone +from html import unescape + +# F LIVE (pas le miroir périmé) — lecture bornée par id seulement. +LEGACY = {"host": os.environ.get("LEGACY_HOST", "10.100.80.100"), "user": "facturation", + "password": os.environ.get("LEGACY_PW", "VD67owoj"), "database": "gestionclient", + "connect_timeout": 30, "read_timeout": 600} +PG = {"host": os.environ.get("PG_HOST", "db"), "port": 5432, "user": "postgres", + "password": os.environ.get("PG_PW", "123"), "dbname": os.environ.get("PG_DB", "_eb65bdc0c4b1b2d6")} + +ADMIN = "Administrator" +COMPANY = "TARGO" +APPLY = os.environ.get("APPLY", "0") == "1" +SINCE = int(os.environ.get("SINCE", "0")) +LIMIT = int(os.environ.get("LIMIT", "0")) # 0 = pas de limite + +def uid(p=""): return p + uuid.uuid4().hex[:10] +def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") +def today(): return datetime.now(timezone.utc).strftime("%Y-%m-%d") +def ts_to_date(t): + if not t or t <= 0: return None + try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d") + except Exception: return None +def clean(v): return unescape(str(v)).strip() if v else "" +def r2(v): + try: return round(float(v or 0), 2) + except Exception: return 0.0 +def sinv_name(i): return "SINV-{:010d}".format(int(i)) +def log(m): print(m, flush=True) + +# F invoice → statut ERPNext reflétant F (= mapInvoice côté hub). +def map_status(total, billed, due, td): + is_return = total < 0 + owed = 0.0 if is_return else r2(max(0.0, total - billed)) + if is_return: status = "Return" + elif owed <= 0.005: status = "Paid" + elif due and due < td: status = "Overdue" + else: status = "Unpaid" + return is_return, owed, status + +def main(): + td = today() + ts = now() + log("=== Sync incrémental factures F→ERPNext (MIROIR OPÉRATIONNEL) ===") + log(" mode = {}".format("APPLY (écriture)" if APPLY else "DRY-RUN (aucune écriture)")) + + pg = psycopg2.connect(**PG) + pgc = pg.cursor() + + # watermark = max legacy_invoice_id déjà dans ERPNext (ou SINCE) + pgc.execute('SELECT COALESCE(MAX(legacy_invoice_id),0) FROM "tabSales Invoice" WHERE legacy_invoice_id > 0') + erp_max = int(pgc.fetchone()[0] or 0) + watermark = SINCE if SINCE > 0 else erp_max + log(" watermark (id F) = {}".format(watermark)) + + # maps ERPNext (mêmes que import_invoices.py) + pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0') + cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()} + pgc.execute('SELECT item_code FROM "tabItem"') + valid_items = set(r[0] for r in pgc.fetchall()) + pgc.execute('SELECT legacy_delivery_id, name FROM "tabService Location" WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id <> 0') + loc_by_delivery = dict(pgc.fetchall()) + pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type='Receivable' AND company=%s AND is_group=0 LIMIT 1", (COMPANY,)) + receivable = pgc.fetchone()[0] + pgc.execute("SELECT account_number, name FROM \"tabAccount\" WHERE root_type IN ('Income','Expense') AND company=%s AND is_group=0 AND account_number <> ''", (COMPANY,)) + gl_by_number = {r[0]: r[1] for r in pgc.fetchall()} + pgc.execute("SELECT name FROM \"tabAccount\" WHERE root_type='Income' AND company=%s AND is_group=0 LIMIT 1", (COMPANY,)) + income_default = pgc.fetchone()[0] + log(" maps: {} clients, {} items, {} locations, {} comptes produit".format(len(cust_map), len(valid_items), len(loc_by_delivery), len(gl_by_number))) + + # ── SYNC PAR NUMÉRO INCRÉMENTIEL, GAP-AWARE & LÉGÈRE ── + # On ne fait PAS un simple `id > max` (qui ignorerait les TROUS sous le max : + # factures sautées « sans client » qui se résolvent plus tard, inserts ratés). + # On diffe les NUMÉROS sur une fenêtre récente (id seulement = ultra-léger), + # et on ne charge/insère QUE les manquants (nouveaux au-dessus du max + trous comblés). + WINDOW = int(os.environ.get("WINDOW", "20000")) # re-vérifie les ~20k derniers numéros + floor = watermark if SINCE > 0 else max(0, watermark - WINDOW) + mc = pymysql.connect(**LEGACY) + cur = mc.cursor(pymysql.cursors.DictCursor) + # 1) numéros F dans la fenêtre (PK indexée, id seul) + cur.execute("SELECT id FROM invoice WHERE id > %s", (floor,)) + f_ids = [r["id"] for r in cur.fetchall()] + # 2) numéros déjà dans ERPNext dans la fenêtre + pgc.execute('SELECT legacy_invoice_id FROM "tabSales Invoice" WHERE legacy_invoice_id > %s', (floor,)) + erp_ids = set(int(r[0]) for r in pgc.fetchall()) + # 3) manquants = à traiter + todo = sorted(i for i in f_ids if i not in erp_ids) + if LIMIT > 0: todo = todo[:LIMIT] + log(" fenêtre id>{} : F={} · ERP={} · à insérer (nouveaux + trous) = {}".format(floor, len(f_ids), len(erp_ids), len(todo))) + + invoices = [] + items_by_inv = {} + for s in range(0, len(todo), 5000): + batch = todo[s:s+5000] + if not batch: continue + ph = ",".join(["%s"]*len(batch)) + cur.execute("SELECT id, account_id, date_orig, due_date, total_amt, billed_amt, billing_status FROM invoice WHERE id IN ({}) ORDER BY id".format(ph), batch) + invoices.extend(cur.fetchall()) + cur.execute("SELECT ii.*, s.delivery_id AS service_delivery_id FROM invoice_item ii LEFT JOIN service s ON s.id=ii.service_id WHERE ii.invoice_id IN ({}) ORDER BY ii.invoice_id, ii.id".format(ph), batch) + for r in cur.fetchall(): + items_by_inv.setdefault(r["invoice_id"], []).append(r) + # mapping SKU→compte produit (product → product_cat.num_compte) + cur.execute("SELECT p.sku, pc.num_compte FROM product p JOIN product_cat pc ON p.category=pc.id WHERE p.sku IS NOT NULL AND pc.num_compte IS NOT NULL") + sku_to_gl = {} + for r in cur.fetchall(): + an = str(int(r["num_compte"])) if r["num_compte"] else None + if an and an in gl_by_number: sku_to_gl[r["sku"]] = gl_by_number[an] + mc.close() + existing = set() # `todo` exclut déjà les présents ; ON CONFLICT couvre toute course + + by_status = {"Paid": 0, "Unpaid": 0, "Overdue": 0, "Return": 0} + ok = skip = no_cust = item_ok = 0 + samples = [] + + for inv in invoices: + if inv["id"] in existing: skip += 1; continue + cd = cust_map.get(inv["account_id"]) + if not cd: no_cust += 1; continue + cust_name, cust_disp = cd + total = r2(inv["total_amt"]); billed = r2(inv["billed_amt"]) + posting = ts_to_date(inv["date_orig"]) or td + due = ts_to_date(inv["due_date"]) or posting + is_return, owed, status = map_status(total, billed, due, td) + by_status[status] += 1 + name = sinv_name(inv["id"]) + if len(samples) < 8: + samples.append({"sinv": name, "status": status, "total": total, "outstanding": owed}) + + if not APPLY: + ok += 1 + item_ok += len(items_by_inv.get(inv["id"], [])) + continue + + try: + pgc.execute(""" + INSERT INTO "tabSales Invoice" ( + name, creation, modified, modified_by, owner, docstatus, idx, + naming_series, customer, customer_name, company, + posting_date, due_date, currency, conversion_rate, + selling_price_list, price_list_currency, + base_grand_total, grand_total, base_net_total, net_total, base_total, total, + outstanding_amount, base_rounded_total, rounded_total, + is_return, is_debit_note, disable_rounded_total, + debit_to, party_account_currency, status, legacy_invoice_id + ) VALUES (%s,%s,%s,%s,%s,1,0, + 'ACC-SINV-.YYYY.-',%s,%s,%s, + %s,%s,'CAD',1, + 'Standard Selling','CAD', + %s,%s,%s,%s,%s,%s, + %s,%s,%s, + %s,0,1, + %s,'CAD',%s,%s) + ON CONFLICT (name) DO NOTHING + """, (name, ts, ts, ADMIN, ADMIN, + cust_name, cust_disp, COMPANY, + posting, due, + total, total, total, total, total, total, + owed, total, total, + 1 if is_return else 0, + receivable, status, inv["id"])) + + for j, li in enumerate(items_by_inv.get(inv["id"], [])): + sku = clean(li.get("sku")) or "MISC" + qty = float(li.get("quantity") or 1) + rate = float(li.get("unitary_price") or 0) + amount = round(qty * rate, 2) + desc = (clean(li.get("product_name")) or sku)[:140] + item_code = sku if sku in valid_items else None + income = sku_to_gl.get(sku, income_default) + sloc = loc_by_delivery.get(li.get("service_delivery_id")) + pgc.execute(""" + INSERT INTO "tabSales Invoice Item" ( + name, creation, modified, modified_by, owner, docstatus, idx, + item_code, item_name, description, qty, rate, amount, + base_rate, base_amount, base_net_rate, base_net_amount, net_rate, net_amount, + stock_uom, uom, conversion_factor, income_account, cost_center, service_location, + parent, parentfield, parenttype + ) VALUES (%s,%s,%s,%s,%s,1,%s, + %s,%s,%s,%s,%s,%s, + %s,%s,%s,%s,%s,%s, + 'Nos','Nos',1,%s,'Main - T',%s, + %s,'items','Sales Invoice') + ON CONFLICT (name) DO NOTHING + """, (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1, + item_code, desc, desc, qty, rate, amount, + rate, amount, rate, amount, rate, amount, + income, sloc, name)) + item_ok += 1 + ok += 1 + except Exception as e: + pg.rollback() + log(" ERR inv#{} -> {}".format(inv["id"], str(e)[:160])) + continue + if ok % 2000 == 0: + pg.commit(); log(" ... {} insérées".format(ok)) + + if APPLY: pg.commit() + pg.close() + log("") + log("="*60) + log("Statut cible (reflète F) : {}".format(by_status)) + log("{} : {} factures, {} lignes, {} déjà présentes, {} sans client".format( + "INSÉRÉES" if APPLY else "À INSÉRER (dry-run)", ok, item_ok, skip, no_cust)) + log("Échantillon : {}".format(samples)) + log("="*60) + +if __name__ == "__main__": + main() diff --git a/scripts/targo-sync/sync_services_incremental.py b/scripts/targo-sync/sync_services_incremental.py new file mode 100644 index 0000000..ef9d4a7 --- /dev/null +++ b/scripts/targo-sync/sync_services_incremental.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +""" +Sync INCRÉMENTAL F → ERPNext — couche OPÉRATIONNELLE adresses + services. + +Crée les enregistrements MANQUANTS (diff complet par clé legacy, petites tables → léger) : + • delivery → Service Location (LOC-{legacy_delivery_id zfill10}) — mapping migrate_locations.py + • service (status=1) → Service Subscription (SUB-{legacy_service_id zfill10}) — mapping import_services_and_enrich_customers.py + +NE crée PAS le doctype standard `Subscription` (moteur de facturation récurrente) : F est +autoritaire pour la facturation (scheduler ERPNext en pause) → couche billing gérée à la bascule, +comme GL/PLE pour les factures. Ici = miroir OPÉRATIONNEL seulement (ce que les équipes voient). + +Ordre : Service Location d'ABORD (un service requiert sa location + le client). Idempotent (ON CONFLICT). +DRY-RUN par défaut ; n'écrit RIEN sans APPLY=1. Options env : LIMIT (lot), PG_HOST. + +Lancer : + docker cp sync_services_incremental.py erpnext-backend-1:/tmp/sync_svc.py + docker exec -e APPLY=0 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py + docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py +""" +import os +import re +import uuid +import pymysql +import psycopg2 +from datetime import datetime, timezone +from html import unescape + +LEGACY = {"host": os.environ.get("LEGACY_HOST", "10.100.80.100"), "user": "facturation", + "password": os.environ.get("LEGACY_PW", "VD67owoj"), "database": "gestionclient", + "connect_timeout": 30, "read_timeout": 600} +PG = {"host": os.environ.get("PG_HOST", "db"), "port": 5432, "user": "postgres", + "password": os.environ.get("PG_PW", "123"), "dbname": os.environ.get("PG_DB", "_eb65bdc0c4b1b2d6")} +ADMIN = "Administrator" +APPLY = os.environ.get("APPLY", "0") == "1" +LIMIT = int(os.environ.get("LIMIT", "0")) + +# ── mappings (identiques à la migration) ── +PROD_CAT_MAP = {4: "Internet", 32: "Internet", 8: "Internet", 26: "Internet", 29: "Internet", + 7: "Internet", 23: "Internet", 17: "Internet", 16: "Internet", 21: "Internet", + 33: "IPTV", 34: "IPTV", 9: "VoIP", 15: "Hébergement", 11: "Hébergement", + 30: "Hébergement", 10: "Autre", 13: "Autre"} +RECUR_MAP = {1: "Mensuel", 2: "Mensuel", 3: "Trimestriel", 4: "Annuel"} + +def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") +def uid(p=""): return p + uuid.uuid4().hex[:10] +def clean(v): return unescape(str(v)).strip() if v else "" +def loc_name(i): return "LOC-{:010d}".format(int(i)) +def sub_name(i): return "SUB-{:010d}".format(int(i)) +def ts_date(t, dflt=None): + if not t or int(t) <= 0: return dflt + try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d") + except Exception: return dflt +def log(m): print(m, flush=True) + +def normalize_postal(pc): # "j0s1b0" → "J0S 1B0" + s = re.sub(r"\s+", "", clean(pc)).upper() + return s[:3] + " " + s[3:6] if len(s) >= 6 else clean(pc) + +def clean_addr(raw, postal=None): # strip postal qui a fui dans address1 + espaces + a = clean(raw) + if postal: + pc = re.sub(r"\s+", "", postal).upper() + a = re.sub(r"\s*" + re.escape(pc[:3]) + r"\s*" + re.escape(pc[3:6]) + r"\s*$", "", a, flags=re.I) + a = re.sub(r"\s*" + re.escape(pc) + r"\s*$", "", a, flags=re.I) + return re.sub(r"\s+", " ", a).strip() + +def svc_status(s, d, resiliated=False): # règle canonique (= legacy-sync.svcStatus) : F service → statut SS + if resiliated: + return "Annulé" # FIX 2026-07-06 : compte F résilié (status 3/4/5) → l'abo ne peut PAS rester Actif. + # F ne cascade PAS la résiliation du compte vers service.status (cf feedback_ghost_active_subscriptions). + if int(s or 0) == 1: + return "Actif" # CORRIGE 2026-06-13: F status=1 = ACTIF (date_suspended = historique, ignore) + return "Annulé" + +def main(): + ts = now() + log("=== Sync incrémental adresses + services (OPÉRATIONNEL, sans Subscription billing) ===") + log(" mode = {}".format("APPLY" if APPLY else "DRY-RUN")) + pg = psycopg2.connect(**PG); pgc = pg.cursor() + mc = pymysql.connect(**LEGACY); cur = mc.cursor(pymysql.cursors.DictCursor) + + # maps communs + pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id>0') + cust_map = {int(r[0]): r[1] for r in pgc.fetchall()} + cur.execute("SELECT id, account_id FROM delivery"); del_acct = {r["id"]: r["account_id"] for r in cur.fetchall()} + # FIX 2026-07-06 — comptes résiliés en F (status 3/4/5) : leurs services restent souvent status=1 + # (F ne cascade pas) → sans ce garde-fou, Phase B/D les (re)mettent 'Actif' = abonnements fantômes. + cur.execute("SELECT id FROM account WHERE status IN (3,4,5)") + resil_acct = set(r["id"] for r in cur.fetchall()) + log(" Comptes F résiliés (status 3/4/5) : {}".format(len(resil_acct))) + + # ═══ PHASE A — Service Location (delivery) ═══ + cur.execute("SELECT id FROM delivery") + f_del = [r["id"] for r in cur.fetchall()] + pgc.execute('SELECT legacy_delivery_id FROM "tabService Location" WHERE legacy_delivery_id>0') + erp_loc = set(int(r[0]) for r in pgc.fetchall()) + miss_loc = sorted(i for i in f_del if i not in erp_loc) + if LIMIT: miss_loc = miss_loc[:LIMIT] + log(" Adresses : F={} ERP={} → {} à créer".format(len(f_del), len(erp_loc), len(miss_loc))) + loc_ok = loc_skip = 0; loc_samples = [] + if miss_loc: + for s in range(0, len(miss_loc), 2000): + batch = miss_loc[s:s+2000] + cur.execute("SELECT id, account_id, name, address1, city, state, zip FROM delivery WHERE id IN ({})".format(",".join(["%s"]*len(batch))), batch) + for d in cur.fetchall(): + postal = normalize_postal(d.get("zip")) + addr = clean_addr(d.get("address1"), postal) + city = clean_addr(d.get("city")) + lname = (clean(d.get("name")) or ("{}, {}".format(addr, city) if addr else "Location-{}".format(d["id"])))[:140] + cust = cust_map.get(d["account_id"]) + if len(loc_samples) < 6: loc_samples.append({"loc": loc_name(d["id"]), "addr": addr, "city": city, "customer": cust}) + if not APPLY: loc_ok += 1; continue + try: + pgc.execute(""" + INSERT INTO "tabService Location" ( + name, creation, modified, modified_by, owner, docstatus, idx, + customer, location_name, status, address_line, city, postal_code, province, legacy_delivery_id + ) VALUES (%s,%s,%s,%s,%s,0,0, %s,%s,'Active',%s,%s,%s,%s,%s) + ON CONFLICT (name) DO NOTHING + """, (loc_name(d["id"]), ts, ts, ADMIN, ADMIN, cust, lname, addr, city, postal, clean(d.get("state")), d["id"])) + loc_ok += 1 + except Exception as e: + loc_skip += 1; pg.rollback() + if loc_skip <= 5: log(" ERR loc#{} -> {}".format(d["id"], str(e)[:120])) + continue + if APPLY: pg.commit() + # recharge le map location pour la phase B + pgc.execute('SELECT legacy_delivery_id, name FROM "tabService Location" WHERE legacy_delivery_id>0') + loc_map = {int(r[0]): r[1] for r in pgc.fetchall()} + + # ═══ PHASE B — Service Subscription (service status=1) ═══ + cur.execute("SELECT id FROM service WHERE status=1") + f_svc = [r["id"] for r in cur.fetchall()] + pgc.execute('SELECT legacy_service_id FROM "tabService Subscription" WHERE legacy_service_id>0') + erp_sub = set(int(r[0]) for r in pgc.fetchall()) + miss_svc = sorted(i for i in f_svc if i not in erp_sub) + if LIMIT: miss_svc = miss_svc[:LIMIT] + log(" Services : F actifs={} ERP={} → {} à créer".format(len(f_svc), len(erp_sub), len(miss_svc))) + # product names FR + cur.execute("SELECT product_id, name AS prod_name FROM product_translate WHERE language_id='francais'") + prod_names = {r["product_id"]: r["prod_name"] for r in cur.fetchall()} + sub_ok = sub_noloc = sub_skip = 0; sub_samples = [] + if miss_svc: + for s in range(0, len(miss_svc), 2000): + batch = miss_svc[s:s+2000] + cur.execute("""SELECT s.id, s.delivery_id, s.product_id, s.payment_recurrence, s.hijack, s.hijack_price, + s.hijack_download_speed, s.hijack_upload_speed, s.date_orig, s.date_end_contract, + s.radius_user, s.radius_pwd, + p.sku, p.price, p.download_speed, p.upload_speed, p.category AS prod_cat + FROM service s LEFT JOIN product p ON s.product_id=p.id + WHERE s.id IN ({})""".format(",".join(["%s"]*len(batch))), batch) + for svc in cur.fetchall(): + sl = loc_map.get(svc["delivery_id"]) if svc["delivery_id"] else None + acct_id = del_acct.get(svc["delivery_id"]) if svc["delivery_id"] else None + cust = cust_map.get(acct_id) if acct_id else None + if not sl or not cust: sub_noloc += 1; continue # location + client requis (idem migration) + new_status = "Annulé" if acct_id in resil_acct else "Actif" # FIX 2026-07-06 : jamais Actif sur compte résilié + cat = PROD_CAT_MAP.get(svc["prod_cat"] or 0, "Autre") + plan = unescape(prod_names.get(svc["product_id"], svc.get("sku") or "Unknown"))[:140] + if svc["hijack"] and svc["hijack_download_speed"]: + sd, su = int(svc["hijack_download_speed"]) // 1024, int(svc["hijack_upload_speed"] or 0) // 1024 + elif svc["download_speed"]: + sd, su = int(svc["download_speed"]) // 1024, int(svc["upload_speed"] or 0) // 1024 + else: sd = su = 0 + price = float(svc["hijack_price"] or 0) if svc["hijack"] else float(svc["price"] or 0) + cycle = RECUR_MAP.get(svc["payment_recurrence"], "Mensuel") + start = ts_date(svc["date_orig"], "2020-01-01"); end = ts_date(svc["date_end_contract"]) + if len(sub_samples) < 6: sub_samples.append({"sub": sub_name(svc["id"]), "cat": cat, "plan": plan, "price": price, "cycle": cycle, "loc": sl}) + if not APPLY: sub_ok += 1; continue + try: + pgc.execute(""" + INSERT INTO "tabService Subscription" ( + name, creation, modified, modified_by, owner, docstatus, idx, + customer, service_location, status, service_category, plan_name, + speed_down, speed_up, monthly_price, billing_cycle, start_date, end_date, + legacy_service_id, radius_user, radius_password, product_sku + ) VALUES (%s,%s,%s,%s,%s,0,0, %s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s, %s,%s,%s,%s) + ON CONFLICT (name) DO NOTHING + """, (sub_name(svc["id"]), ts, ts, ADMIN, ADMIN, cust, sl, new_status, cat, plan, + sd, su, price, cycle, start, end, + svc["id"], clean(svc.get("radius_user")), clean(svc.get("radius_pwd")), clean(svc.get("sku")))) + sub_ok += 1 + except Exception as e: + sub_skip += 1; pg.rollback() + if sub_skip <= 5: log(" ERR svc#{} -> {}".format(svc["id"], str(e)[:120])) + continue + if APPLY: pg.commit() + + # ═══ PHASE C — rafraîchir le PRIX des SS existantes (F autoritaire ; opérationnel) ═══ + # Les prix F (rabais/crédits/forfaits) bougent après la migration → le monthly_price figé + # de la SS diverge. On le réaligne sur F. (Statut Actif/Suspendu/Annulé = legacy-sync.syncServices.) + cur.execute("SELECT s.id, IF(s.hijack=1, COALESCE(s.hijack_price,0), COALESCE(p.price,0)) AS fprice FROM service s LEFT JOIN product p ON p.id=s.product_id WHERE s.status=1") + fprice = {r["id"]: round(float(r["fprice"]), 2) for r in cur.fetchall()} + pgc.execute('SELECT legacy_service_id, monthly_price FROM "tabService Subscription" WHERE legacy_service_id>0') + to_upd = [] + for lid, mp in pgc.fetchall(): + f = fprice.get(int(lid)) + if f is None: continue + if abs(round(float(mp or 0), 2) - f) > 0.011: to_upd.append((f, int(lid))) + price_fixed = 0 + log(" Prix SS divergents vs F : {}".format(len(to_upd))) + if APPLY and to_upd: + for f, lid in to_upd: + pgc.execute('UPDATE "tabService Subscription" SET monthly_price=%s, modified=%s WHERE legacy_service_id=%s AND ROUND(monthly_price::numeric,2)<>%s', (f, ts, lid, f)) + pg.commit(); price_fixed = len(to_upd) + + # ═══ PHASE D — rafraîchir le STATUT des SS (Actif/Suspendu/Annulé) depuis F ═══ + # Reflète résiliations (status 1→0 = Annulé) + suspensions (date_suspended>0 = Suspendu) + réactivations. + # C'est la SOURCE du statut client (disabled dérivé des services — cf. feedback_customer_status_policy). + cur.execute("SELECT s.id, s.status, s.date_suspended, d.account_id FROM service s LEFT JOIN delivery d ON d.id=s.delivery_id") + want = {r["id"]: svc_status(r["status"], r["date_suspended"], r["account_id"] in resil_acct) for r in cur.fetchall()} + pgc.execute('SELECT legacy_service_id, status FROM "tabService Subscription" WHERE legacy_service_id>0') + st_upd = []; trans = {} + for lid, st in pgc.fetchall(): + w = want.get(int(lid)) + if w and w != st: + st_upd.append((w, int(lid))); k = (st or "∅") + "→" + w; trans[k] = trans.get(k, 0) + 1 + status_fixed = 0 + log(" Statut SS divergents vs F : {} {}".format(len(st_upd), trans)) + if APPLY and st_upd: + for w, lid in st_upd: + pgc.execute('UPDATE "tabService Subscription" SET status=%s, modified=%s WHERE legacy_service_id=%s AND status<>%s', (w, ts, lid, w)) + pg.commit(); status_fixed = len(st_upd) + + mc.close(); pg.close() + log("") + log("=" * 60) + log("Statut SS réalignés sur F : {} {}".format(status_fixed if APPLY else len(st_upd), "corrigés" if APPLY else "à corriger (dry-run)")) + log("Prix SS réalignés sur F : {} {}".format(price_fixed if APPLY else len(to_upd), "corrigés" if APPLY else "à corriger (dry-run)")) + log("Adresses : {} {} | {} err".format(loc_ok, "créées" if APPLY else "à créer (dry-run)", loc_skip)) + log(" échantillon: {}".format(loc_samples)) + log("Services : {} {} | {} sans location/client | {} err".format(sub_ok, "créés" if APPLY else "à créer (dry-run)", sub_noloc, sub_skip)) + log(" échantillon: {}".format(sub_samples)) + log("=" * 60) + +if __name__ == "__main__": + main() diff --git a/services/targo-hub/lib/legacy-sync.js b/services/targo-hub/lib/legacy-sync.js index 3ad54f9..ba553ff 100644 --- a/services/targo-hub/lib/legacy-sync.js +++ b/services/targo-hub/lib/legacy-sync.js @@ -501,6 +501,78 @@ async function syncServices ({ confirm = false } = {}) { return { ...plan, applied: ok, errors: err, error_samples: errs } } +// ── CORRECTION : abonnements « fantômes » sur comptes F RÉSILIÉS ───────────── +// Bug : F résilie au niveau COMPTE (account.status 3/4/5) mais laisse service.status=1 → le +// mapping svcStatus (basé sur service.status seul) garde l'abonnement 'Actif'. Ici on force +// 'Annulé' pour tout abonnement 'Actif' dont le client a un legacy_account_id RÉSILIÉ. +// Robuste (indépendant du bridge : fResiliatedAccountIds + jointure PG). Idempotent → réexécutable +// périodiquement (fix « forward »). dry-run sauf confirm==='F-WINS'. Cf feedback_ghost_active_subscriptions. +async function annulResiliatedSubscriptions ({ confirm = false, limit = 20000 } = {}) { + const resil = await fResiliatedAccountIds() + if (!resil || !resil.length) return { resiliated_accounts: 0, would_update: 0 } + const pg = require('./address-db').pool() + const rows = [] + for (let i = 0; i < resil.length; i += 2000) { + const b = resil.slice(i, i + 2000) + const r = await pg.query('SELECT ss.name, ss.customer, ss.monthly_price FROM "tabService Subscription" ss JOIN "tabCustomer" c ON ss.customer=c.name WHERE ss.status=$1 AND c.legacy_account_id = ANY($2)', ['Actif', b]) + rows.push(...r.rows) + } + const plan = { resiliated_accounts: resil.length, would_update: rows.length, monthly_sum: Math.round(rows.reduce((s, x) => s + (Number(x.monthly_price) || 0), 0)) } + if (confirm !== 'F-WINS') return { dry_run: true, ...plan, sample: rows.slice(0, 15).map(x => ({ name: x.name, customer: x.customer })) } + let ok = 0; let err = 0; const errs = []; let i = 0 + for (const x of rows.slice(0, limit)) { + try { const r = await withRetry(() => erp.update('Service Subscription', x.name, { status: 'Annulé' }), 'annul ' + x.name); if (r && r.ok !== false) ok++; else { err++; if (errs.length < 15) errs.push({ name: x.name, err: (r && r.error) || 'fail' }) } } + catch (e) { err++; if (errs.length < 15) errs.push({ name: x.name, err: e.message }) } + if (++i % 25 === 0) await new Promise(r => setTimeout(r, 200)) // throttle + } + log(`legacy-sync annulResiliatedSubscriptions: ${ok} annulés, ${err} err`) + return { ...plan, applied: ok, errors: err, error_samples: errs } +} + +// ── Tableau de bord de réconciliation : F (mysql) vs ERPNext (pg), par entité ── +// Chaque compte est indépendant (try/catch → null si table absente) pour ne jamais casser la vue. +async function scoreboard () { + const pg = require('./address-db').pool() + const pgc = async (sql, params) => { try { return Number((await pg.query(sql, params)).rows[0].c) } catch (e) { return null } } + const mp = pool() + const myc = async (sql) => { if (!mp) return null; try { const [r] = await mp.query({ sql, timeout: 8000 }); return Number(r[0].c) } catch (e) { return null } } + let resil = []; try { resil = await fResiliatedAccountIds() } catch (e) {} + const [fAcct, fSvcActive, fDelivery, fTicket, fInvoice, fPayment, fDevice, + eCust, eSubActif, eLoc, eIssue, eInv, ePay, eDevice, ghost] = await Promise.all([ + myc('SELECT COUNT(*) c FROM account'), + // F actif = service.status=1 SUR compte non-résilié (F ne cascade pas la résiliation → comptage brut gonflé) : comparé pomme-à-pomme au 'Actif' ERPNext maintenant nettoyé + myc('SELECT COUNT(*) c FROM service s JOIN delivery d ON d.id=s.delivery_id JOIN account a ON a.id=d.account_id WHERE s.status=1 AND a.status NOT IN (3,4,5)'), + myc('SELECT COUNT(*) c FROM delivery'), + myc('SELECT COUNT(*) c FROM ticket'), + myc('SELECT COUNT(*) c FROM invoice'), + myc('SELECT COUNT(*) c FROM payment'), + myc('SELECT COUNT(*) c FROM device'), + pgc('SELECT COUNT(*) c FROM "tabCustomer"'), + pgc('SELECT COUNT(*) c FROM "tabService Subscription" WHERE status=$1', ['Actif']), + pgc('SELECT COUNT(*) c FROM "tabService Location"'), + pgc('SELECT COUNT(*) c FROM "tabIssue"'), + pgc('SELECT COUNT(*) c FROM "tabSales Invoice"'), + pgc('SELECT COUNT(*) c FROM "tabPayment Entry"'), + pgc('SELECT COUNT(*) c FROM "tabService Equipment"'), + resil.length ? pgc('SELECT COUNT(*) c FROM "tabService Subscription" ss JOIN "tabCustomer" c ON ss.customer=c.name WHERE ss.status=$1 AND c.legacy_account_id = ANY($2)', ['Actif', resil]) : 0, + ]) + const row = (key, label, f, erp, note) => ({ key, label, f, erp, delta: (f != null && erp != null) ? f - erp : null, note: note || '' }) + return { + generated_at: new Date().toISOString(), + resiliated_f_accounts: resil.length, + ghost_active_subscriptions: ghost, + entities: [ + row('customers', 'Clients', fAcct, eCust, 'F account ↔ Customer'), + row('locations', 'Lieux de service', fDelivery, eLoc, 'F delivery ↔ Service Location'), + row('subscriptions', 'Abonnements actifs', fSvcActive, eSubActif, ghost ? `${ghost} fantômes (compte résilié) à annuler` : 'F actif hors compte résilié ↔ Actif'), + row('devices', 'Appareils', fDevice, eDevice, 'F device ↔ Service Equipment (liens orphelins possibles)'), + row('tickets', 'Tickets', fTicket, eIssue, 'F ticket ↔ Issue (import à réparer)'), + row('invoices', 'Factures', fInvoice, eInv, 'F invoice ↔ Sales Invoice'), + row('payments', 'Paiements', fPayment, ePay, 'F payment ↔ Payment Entry'), + ], + } +} + // ── CRÉATEUR DE CLIENTS CANONIQUE (utilisé par billing + orchestrateur) ────── // Crée les Customers ERPNext manquants depuis F (par legacy_account_id) via erp.create // (REST → naming/validations/defaults gérés). Idempotent. POLITIQUE : créé VISIBLE @@ -734,6 +806,7 @@ async function handle (req, res, method, p, url) { try { return json(res, 200, await bridgeCall({ action: 'ping' })) } catch (e) { return json(res, 502, { error: e.message }) } } + if (p === '/legacy-sync/scoreboard' && method === 'GET') { try { return json(res, 200, await scoreboard()) } catch (e) { return json(res, 500, { error: e.message }) } } if (p === '/legacy-sync/state' && method === 'GET') return json(res, 200, loadState()) if (p === '/legacy-sync/run-status' && method === 'GET') { try { return json(res, 200, JSON.parse(fs.readFileSync(RUN_LOG, 'utf8'))) } catch { return json(res, 200, { status: 'idle' }) } } // Liste COMPLÈTE des erreurs du dernier apply, groupées par raison. ?reason=email_invalide pour filtrer. @@ -748,4 +821,4 @@ async function handle (req, res, method, p, url) { return json(res, 404, { error: 'route legacy-sync inconnue' }) } -module.exports = { handle, runPreview, previewCustomers, applySafeAdds, syncCustomers, syncServices, createCustomers, createCustomersByIds, applyAccountChange, applyReviewBatch, fCounts, domainCounts, loadState, mapAccount, svcStatus, fAccountStatuses, fResiliatedAccountIds, F_ACCT_STATUS_LABEL, GROUP_MAP, CMP } +module.exports = { handle, runPreview, previewCustomers, applySafeAdds, syncCustomers, syncServices, annulResiliatedSubscriptions, scoreboard, createCustomers, createCustomersByIds, applyAccountChange, applyReviewBatch, fCounts, domainCounts, loadState, mapAccount, svcStatus, fAccountStatuses, fResiliatedAccountIds, F_ACCT_STATUS_LABEL, GROUP_MAP, CMP }