sync-legacy: scoreboard de réconciliation + fix des abonnements fantômes à la source
Scoreboard (console de réconciliation F ↔ ERPNext) :
- hub: legacy-sync.js scoreboard() + GET /legacy-sync/scoreboard — comptes bruts
par module (clients/lieux/abos/appareils/tickets/factures/paiements) + métrique
ghost_active_subscriptions. Devices = tabService Equipment (pas tabDevice).
Count F « abos actifs » exclut les comptes résiliés (comparaison pomme-à-pomme).
- ops: carte « Réconciliation par module » sur LegacySyncPage.vue.
Fix fantômes à la SOURCE (scripts/targo-sync/, jusqu'ici hors repo) :
- La vraie sync des services = scripts Python hôte (/opt/targo-sync/, cron horaire),
PAS le hub. svc_status() ignorait le statut du COMPTE → un compte résilié F
(status 3/4/5) dont les services restent status=1 ressortait « Actif » (F ne
cascade pas). Résultat : ~3855 abonnements fantômes recréés à chaque heure.
- sync_services_incremental.py: svc_status() rendu conscient du compte (force
'Annulé' si compte résilié) sur Phase B (création) + Phase D (rafraîchissement).
Rollout: dry-run {Actif→Annulé: 3855} → APPLY=1 → ghost=0 stable.
- Versionne aussi run.sh + sync_invoices_incremental.py + README (snapshot fidèle
de la prod ; ces scripts écrivent en base chaque heure et n'étaient pas suivis).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
1b6abc43f2
commit
004c3f8dee
|
|
@ -32,6 +32,51 @@
|
||||||
</q-card-section>
|
</q-card-section>
|
||||||
</q-card>
|
</q-card>
|
||||||
|
|
||||||
|
<!-- ── TABLEAU DE BORD DE RÉCONCILIATION : comptes bruts F vs ERPNext, par module ── -->
|
||||||
|
<q-card flat bordered class="q-mb-md">
|
||||||
|
<q-card-section class="row items-center q-py-sm">
|
||||||
|
<q-icon name="balance" size="20px" color="grey-7" class="q-mr-sm" />
|
||||||
|
<div>
|
||||||
|
<div class="text-weight-bold">Réconciliation par module — F vs ERPNext</div>
|
||||||
|
<div class="text-caption text-grey-6">Comptes bruts de chaque côté. Écart = F − ERPNext (positif = manque dans ERPNext).</div>
|
||||||
|
</div>
|
||||||
|
<q-space />
|
||||||
|
<q-btn flat dense round size="sm" icon="refresh" :loading="sbLoading" @click="loadScoreboard"><q-tooltip>Recalculer les écarts</q-tooltip></q-btn>
|
||||||
|
</q-card-section>
|
||||||
|
<q-markup-table v-if="scoreboard" flat dense wrap-cells separator="horizontal">
|
||||||
|
<thead>
|
||||||
|
<tr class="text-grey-7">
|
||||||
|
<th class="text-left">Module</th>
|
||||||
|
<th class="text-right">F (legacy)</th>
|
||||||
|
<th class="text-right">ERPNext</th>
|
||||||
|
<th class="text-right">Écart</th>
|
||||||
|
<th class="text-left" style="min-width:180px">Note</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="e in scoreboard.entities" :key="e.key">
|
||||||
|
<td class="text-left text-weight-medium">{{ e.label }}</td>
|
||||||
|
<td class="text-right">{{ fmtN(e.f) }}</td>
|
||||||
|
<td class="text-right">{{ fmtN(e.erp) }}</td>
|
||||||
|
<td class="text-right text-weight-bold" :class="deltaClass(e.delta)">
|
||||||
|
<q-icon v-if="e.delta === 0" name="check_circle" size="14px" class="q-mr-xs" />{{ e.delta == null ? 'n/a' : (e.delta > 0 ? '+' : '') + fmtN(e.delta) }}
|
||||||
|
</td>
|
||||||
|
<td class="text-left text-caption text-grey-7">{{ e.note }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</q-markup-table>
|
||||||
|
<q-card-section v-if="scoreboard" class="q-pt-sm text-caption text-grey-6 row items-center q-gutter-x-md">
|
||||||
|
<span>{{ fmtN(scoreboard.resiliated_f_accounts) }} comptes F résiliés</span>
|
||||||
|
<span>·</span>
|
||||||
|
<span :class="scoreboard.ghost_active_subscriptions ? 'text-orange-9 text-weight-bold' : 'text-green-8'">
|
||||||
|
<q-icon :name="scoreboard.ghost_active_subscriptions ? 'warning' : 'check_circle'" size="14px" /> {{ fmtN(scoreboard.ghost_active_subscriptions) }} abonnements fantômes (compte résilié)
|
||||||
|
</span>
|
||||||
|
<q-space />
|
||||||
|
<span v-if="scoreboard.generated_at">{{ new Date(scoreboard.generated_at).toLocaleString('fr-CA') }}</span>
|
||||||
|
</q-card-section>
|
||||||
|
<q-card-section v-else class="text-caption text-grey-6"><q-spinner size="16px" class="q-mr-xs" /> Calcul des écarts par module…</q-card-section>
|
||||||
|
</q-card>
|
||||||
|
|
||||||
<!-- ── BANDE UNIFIÉE : tous les domaines en ordre de dépendance (DAG) ── -->
|
<!-- ── BANDE UNIFIÉE : tous les domaines en ordre de dépendance (DAG) ── -->
|
||||||
<q-card flat bordered class="q-mb-md">
|
<q-card flat bordered class="q-mb-md">
|
||||||
<q-card-section class="row items-center q-py-sm">
|
<q-card-section class="row items-center q-py-sm">
|
||||||
|
|
@ -197,6 +242,9 @@ const sd = computed(() => (rep.value && rep.value.customers.status_divergence) |
|
||||||
// ── Bande unifiée (rapide, /sync/status) ──
|
// ── Bande unifiée (rapide, /sync/status) ──
|
||||||
const sLoading = ref(false)
|
const sLoading = ref(false)
|
||||||
const status = ref(null)
|
const status = ref(null)
|
||||||
|
// ── Tableau de bord de réconciliation (comptes bruts F vs ERPNext, /legacy-sync/scoreboard) ──
|
||||||
|
const scoreboard = ref(null)
|
||||||
|
const sbLoading = ref(false)
|
||||||
const health = computed(() => (status.value && status.value.billing_health) || {})
|
const health = computed(() => (status.value && status.value.billing_health) || {})
|
||||||
const fmtN = (v) => v == null ? '—' : Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 })
|
const fmtN = (v) => v == null ? '—' : Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 })
|
||||||
const fmtCur = (v) => v == null ? '—' : '$' + Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 })
|
const fmtCur = (v) => v == null ? '—' : '$' + Number(v).toLocaleString('fr-CA', { maximumFractionDigits: 0 })
|
||||||
|
|
@ -230,7 +278,12 @@ async function load () {
|
||||||
} catch (e) { err.value = e.message }
|
} catch (e) { err.value = e.message }
|
||||||
loading.value = false
|
loading.value = false
|
||||||
}
|
}
|
||||||
function reload () { loadStatus(); load() }
|
async function loadScoreboard () {
|
||||||
|
sbLoading.value = true
|
||||||
|
try { const r = await fetch(`${HUB_URL}/legacy-sync/scoreboard`); if (r.ok) scoreboard.value = await r.json() } catch {} finally { sbLoading.value = false }
|
||||||
|
}
|
||||||
|
function deltaClass (d) { if (d == null) return 'text-grey-5'; if (d === 0) return 'text-green-7'; if (Math.abs(d) < 100) return 'text-orange-8'; return 'text-red-7' }
|
||||||
|
function reload () { loadStatus(); load(); loadScoreboard() }
|
||||||
|
|
||||||
// ── Synchronisation manuelle + progression ──
|
// ── Synchronisation manuelle + progression ──
|
||||||
const syncing = ref(false)
|
const syncing = ref(false)
|
||||||
|
|
|
||||||
59
scripts/targo-sync/README.md
Normal file
59
scripts/targo-sync/README.md
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
# targo-sync — miroir opérationnel F (legacy) → ERPNext
|
||||||
|
|
||||||
|
Snapshot versionné des scripts qui tournent sur **l'hôte de prod** dans `/opt/targo-sync/`
|
||||||
|
(hors conteneur). Avant 2026-07-06 ils n'étaient dans aucun repo — c'est le vrai moteur de
|
||||||
|
synchronisation « legacy F → ERPNext » pour la couche **opérationnelle** (ce que les équipes
|
||||||
|
voient : clients, adresses, services/abonnements, factures, paiements). **F reste autoritaire**
|
||||||
|
pour la facturation (scheduler ERPNext en pause) ; ERPNext = miroir + grand livre à la bascule.
|
||||||
|
|
||||||
|
## Cadence
|
||||||
|
|
||||||
|
Cron horaire sur l'hôte (`15 * * * *`) → `run.sh` → 4 étapes, dans l'ordre du DAG de dépendances :
|
||||||
|
|
||||||
|
1. **Comptes manquants** — hub `POST /legacy-payments/ensure-customers` (crée les Customer absents).
|
||||||
|
2. **Adresses + services** — `sync_services_incremental.py` (Python, exécuté **dans** `erpnext-backend-1`).
|
||||||
|
3. **Factures** — `sync_invoices_incremental.py` (idem).
|
||||||
|
4. **Paiements + soldes** — hub `POST /legacy-payments/sync-cycle`.
|
||||||
|
|
||||||
|
Les scripts Python sont idempotents (`ON CONFLICT DO NOTHING` pour les créations, `UPDATE`
|
||||||
|
conditionnel pour les rafraîchissements). **`APPLY=0` = dry-run** (n'écrit rien) ; `run.sh` lance
|
||||||
|
en `APPLY=1`. Autres env : `PG_HOST` (défaut `db`), `LIMIT` (taille de lot), `LEGACY_HOST`.
|
||||||
|
|
||||||
|
## Déploiement (host, hors repo)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
scp scripts/targo-sync/*.py scripts/targo-sync/run.sh root@<prod>:/opt/targo-sync/
|
||||||
|
# le cron docker cp le .py dans erpnext-backend-1 puis l'exécute — pas de restart requis
|
||||||
|
```
|
||||||
|
|
||||||
|
Dry-run manuel d'un script avant bascule :
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker cp /opt/targo-sync/sync_services_incremental.py erpnext-backend-1:/tmp/svc.py
|
||||||
|
docker exec -e APPLY=0 -e PG_HOST=db erpnext-backend-1 \
|
||||||
|
/home/frappe/frappe-bench/env/bin/python /tmp/svc.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## ⚠️ FIX 2026-07-06 — abonnements fantômes (compte résilié affiché « Actif »)
|
||||||
|
|
||||||
|
En F, résilier un **compte** met `account.status ∈ {3,4,5}` mais **laisse les services à
|
||||||
|
`service.status=1`** (F ne cascade pas). `svc_status()` dérivait le statut du seul
|
||||||
|
`service.status` → un compte résilié ressortait « Actif » dans ERPNext. Comme le statut client
|
||||||
|
est dérivé des abonnements, ~2 663 ex-clients apparaissaient actifs et ~3 855 abonnements
|
||||||
|
étaient des fantômes (MRR gonflé, audiences polluées).
|
||||||
|
|
||||||
|
Correctif dans `sync_services_incremental.py` (marqueur `FIX 2026-07-06`) : `svc_status()` charge
|
||||||
|
une fois `SELECT id FROM account WHERE status IN (3,4,5)` et **force `Annulé` dès que le compte est
|
||||||
|
résilié**, sur les deux chemins (Phase B création + Phase D rafraîchissement). Sans ce garde-fou,
|
||||||
|
le cron ré-activait les fantômes à chaque heure. Voir la mémoire projet `feedback_ghost_active_subscriptions`.
|
||||||
|
|
||||||
|
Réconciliation visible dans OPS : page **/sync-legacy** → carte « Réconciliation par module »
|
||||||
|
(endpoint hub `GET /legacy-sync/scoreboard`, métrique `ghost_active_subscriptions` = doit rester 0).
|
||||||
|
|
||||||
|
## Fichiers
|
||||||
|
|
||||||
|
| Fichier | Rôle |
|
||||||
|
|---|---|
|
||||||
|
| `run.sh` | Orchestrateur cron (4 étapes DAG) |
|
||||||
|
| `sync_services_incremental.py` | Service Location (delivery) + Service Subscription (service) : création + prix + **statut (fix fantômes)** |
|
||||||
|
| `sync_invoices_incremental.py` | Sales Invoice (miroir des factures F ; statut reflétant F) |
|
||||||
26
scripts/targo-sync/run.sh
Executable file
26
scripts/targo-sync/run.sh
Executable file
|
|
@ -0,0 +1,26 @@
|
||||||
|
#!/bin/bash
|
||||||
|
LOG=/opt/targo-sync/billing-sync.log; [ -f "$LOG" ] && tail -n 1500 "$LOG" > "$LOG.tmp" 2>/dev/null && mv "$LOG.tmp" "$LOG"
|
||||||
|
# Sync F→ERPNext (miroir opérationnel) — idempotent, léger, gap-aware. Ordre = DAG des dépendances.
|
||||||
|
set -uo pipefail
|
||||||
|
LOG=/opt/targo-sync/billing-sync.log
|
||||||
|
exec >> "$LOG" 2>&1
|
||||||
|
echo "===== $(date -Is) ====="
|
||||||
|
echo "-- 1. comptes manquants (hub) --"
|
||||||
|
docker exec targo-hub node -e '
|
||||||
|
const T=process.env.HUB_SERVICE_TOKEN
|
||||||
|
fetch("http://localhost:3300/legacy-payments/ensure-customers",{method:"POST",headers:{Authorization:"Bearer "+T,"Content-Type":"application/json"},body:JSON.stringify({confirm:"F-WINS",limit:1000})})
|
||||||
|
.then(r=>r.json()).then(d=>console.log("comptes:",JSON.stringify({missing:d.missing,created:d.created,errors:d.errors}))).catch(e=>console.log("comptes ERR",e.message))
|
||||||
|
'
|
||||||
|
echo "-- 2. adresses + services (Python, opérationnel) --"
|
||||||
|
docker cp /opt/targo-sync/sync_services_incremental.py erpnext-backend-1:/tmp/sync_svc.py 2>/dev/null
|
||||||
|
docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py 2>&1 | tail -4
|
||||||
|
echo "-- 3. factures (Python, statut reflétant F) --"
|
||||||
|
docker cp /opt/targo-sync/sync_invoices_incremental.py erpnext-backend-1:/tmp/sync_inv.py 2>/dev/null
|
||||||
|
docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py 2>&1 | tail -2
|
||||||
|
echo "-- 4. paiements + soldes (hub) --"
|
||||||
|
docker exec targo-hub node -e '
|
||||||
|
const T=process.env.HUB_SERVICE_TOKEN
|
||||||
|
fetch("http://localhost:3300/legacy-payments/sync-cycle",{method:"POST",headers:{Authorization:"Bearer "+T,"Content-Type":"application/json"},body:JSON.stringify({confirm:"F-WINS",limit:20000})})
|
||||||
|
.then(r=>r.json()).then(d=>console.log("cycle:",JSON.stringify({pay:d.payments&&d.payments.applied,refreshed:d.open_refresh&&d.open_refresh.updated}))).catch(e=>{console.log("cycle ERR",e.message)})
|
||||||
|
'
|
||||||
|
echo "done $(date -Is)"
|
||||||
233
scripts/targo-sync/sync_invoices_incremental.py
Normal file
233
scripts/targo-sync/sync_invoices_incremental.py
Normal file
|
|
@ -0,0 +1,233 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Sync INCRÉMENTAL des factures F (gestionclient) → ERPNext — MIROIR OPÉRATIONNEL.
|
||||||
|
|
||||||
|
Insère les factures F absentes d'ERPNext (id > watermark) en REFLÉTANT leur statut F :
|
||||||
|
• Sales Invoice + Sales Invoice Item (mêmes maps que import_invoices.py → cohérence
|
||||||
|
exacte avec les ~629k déjà importées : client, SKU→compte produit, service_location).
|
||||||
|
• docstatus = 1 (submitted), status = Paid/Unpaid/Overdue/Return d'après F,
|
||||||
|
outstanding_amount = max(0, total_amt − billed_amt) (F = vérité du solde).
|
||||||
|
• SINV-{legacy_invoice_id zfill10} (idempotent : saute si déjà présent).
|
||||||
|
• PAS de GL Entry / PLE (ERPNext non autoritaire — miroir opérationnel, scheduler en pause).
|
||||||
|
|
||||||
|
LÉGER SUR F : ne lit que `WHERE id > watermark` (PK indexée) sur F LIVE. Aucun scan complet.
|
||||||
|
DRY-RUN par défaut : n'écrit RIEN sans la variable d'env APPLY=1.
|
||||||
|
|
||||||
|
Lancer (depuis l'hôte prod qui joint F live 10.100.80.100 + le PG ERPNext) :
|
||||||
|
docker cp sync_invoices_incremental.py erpnext-backend-1:/tmp/sync_inv.py
|
||||||
|
docker exec -e APPLY=0 erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py # aperçu
|
||||||
|
docker exec -e APPLY=1 erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_inv.py # écrit
|
||||||
|
Options env : SINCE (force le plancher d'id), LIMIT (taille du lot, 0=tout).
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
|
# F LIVE (pas le miroir périmé) — lecture bornée par id seulement.
|
||||||
|
LEGACY = {"host": os.environ.get("LEGACY_HOST", "10.100.80.100"), "user": "facturation",
|
||||||
|
"password": os.environ.get("LEGACY_PW", "VD67owoj"), "database": "gestionclient",
|
||||||
|
"connect_timeout": 30, "read_timeout": 600}
|
||||||
|
PG = {"host": os.environ.get("PG_HOST", "db"), "port": 5432, "user": "postgres",
|
||||||
|
"password": os.environ.get("PG_PW", "123"), "dbname": os.environ.get("PG_DB", "_eb65bdc0c4b1b2d6")}
|
||||||
|
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
COMPANY = "TARGO"
|
||||||
|
APPLY = os.environ.get("APPLY", "0") == "1"
|
||||||
|
SINCE = int(os.environ.get("SINCE", "0"))
|
||||||
|
LIMIT = int(os.environ.get("LIMIT", "0")) # 0 = pas de limite
|
||||||
|
|
||||||
|
def uid(p=""): return p + uuid.uuid4().hex[:10]
|
||||||
|
def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
def today(): return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
def ts_to_date(t):
|
||||||
|
if not t or t <= 0: return None
|
||||||
|
try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
except Exception: return None
|
||||||
|
def clean(v): return unescape(str(v)).strip() if v else ""
|
||||||
|
def r2(v):
|
||||||
|
try: return round(float(v or 0), 2)
|
||||||
|
except Exception: return 0.0
|
||||||
|
def sinv_name(i): return "SINV-{:010d}".format(int(i))
|
||||||
|
def log(m): print(m, flush=True)
|
||||||
|
|
||||||
|
# F invoice → statut ERPNext reflétant F (= mapInvoice côté hub).
|
||||||
|
def map_status(total, billed, due, td):
|
||||||
|
is_return = total < 0
|
||||||
|
owed = 0.0 if is_return else r2(max(0.0, total - billed))
|
||||||
|
if is_return: status = "Return"
|
||||||
|
elif owed <= 0.005: status = "Paid"
|
||||||
|
elif due and due < td: status = "Overdue"
|
||||||
|
else: status = "Unpaid"
|
||||||
|
return is_return, owed, status
|
||||||
|
|
||||||
|
def main():
|
||||||
|
td = today()
|
||||||
|
ts = now()
|
||||||
|
log("=== Sync incrémental factures F→ERPNext (MIROIR OPÉRATIONNEL) ===")
|
||||||
|
log(" mode = {}".format("APPLY (écriture)" if APPLY else "DRY-RUN (aucune écriture)"))
|
||||||
|
|
||||||
|
pg = psycopg2.connect(**PG)
|
||||||
|
pgc = pg.cursor()
|
||||||
|
|
||||||
|
# watermark = max legacy_invoice_id déjà dans ERPNext (ou SINCE)
|
||||||
|
pgc.execute('SELECT COALESCE(MAX(legacy_invoice_id),0) FROM "tabSales Invoice" WHERE legacy_invoice_id > 0')
|
||||||
|
erp_max = int(pgc.fetchone()[0] or 0)
|
||||||
|
watermark = SINCE if SINCE > 0 else erp_max
|
||||||
|
log(" watermark (id F) = {}".format(watermark))
|
||||||
|
|
||||||
|
# maps ERPNext (mêmes que import_invoices.py)
|
||||||
|
pgc.execute('SELECT legacy_account_id, name, customer_name FROM "tabCustomer" WHERE legacy_account_id > 0')
|
||||||
|
cust_map = {r[0]: (r[1], r[2]) for r in pgc.fetchall()}
|
||||||
|
pgc.execute('SELECT item_code FROM "tabItem"')
|
||||||
|
valid_items = set(r[0] for r in pgc.fetchall())
|
||||||
|
pgc.execute('SELECT legacy_delivery_id, name FROM "tabService Location" WHERE legacy_delivery_id IS NOT NULL AND legacy_delivery_id <> 0')
|
||||||
|
loc_by_delivery = dict(pgc.fetchall())
|
||||||
|
pgc.execute("SELECT name FROM \"tabAccount\" WHERE account_type='Receivable' AND company=%s AND is_group=0 LIMIT 1", (COMPANY,))
|
||||||
|
receivable = pgc.fetchone()[0]
|
||||||
|
pgc.execute("SELECT account_number, name FROM \"tabAccount\" WHERE root_type IN ('Income','Expense') AND company=%s AND is_group=0 AND account_number <> ''", (COMPANY,))
|
||||||
|
gl_by_number = {r[0]: r[1] for r in pgc.fetchall()}
|
||||||
|
pgc.execute("SELECT name FROM \"tabAccount\" WHERE root_type='Income' AND company=%s AND is_group=0 LIMIT 1", (COMPANY,))
|
||||||
|
income_default = pgc.fetchone()[0]
|
||||||
|
log(" maps: {} clients, {} items, {} locations, {} comptes produit".format(len(cust_map), len(valid_items), len(loc_by_delivery), len(gl_by_number)))
|
||||||
|
|
||||||
|
# ── SYNC PAR NUMÉRO INCRÉMENTIEL, GAP-AWARE & LÉGÈRE ──
|
||||||
|
# On ne fait PAS un simple `id > max` (qui ignorerait les TROUS sous le max :
|
||||||
|
# factures sautées « sans client » qui se résolvent plus tard, inserts ratés).
|
||||||
|
# On diffe les NUMÉROS sur une fenêtre récente (id seulement = ultra-léger),
|
||||||
|
# et on ne charge/insère QUE les manquants (nouveaux au-dessus du max + trous comblés).
|
||||||
|
WINDOW = int(os.environ.get("WINDOW", "20000")) # re-vérifie les ~20k derniers numéros
|
||||||
|
floor = watermark if SINCE > 0 else max(0, watermark - WINDOW)
|
||||||
|
mc = pymysql.connect(**LEGACY)
|
||||||
|
cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
# 1) numéros F dans la fenêtre (PK indexée, id seul)
|
||||||
|
cur.execute("SELECT id FROM invoice WHERE id > %s", (floor,))
|
||||||
|
f_ids = [r["id"] for r in cur.fetchall()]
|
||||||
|
# 2) numéros déjà dans ERPNext dans la fenêtre
|
||||||
|
pgc.execute('SELECT legacy_invoice_id FROM "tabSales Invoice" WHERE legacy_invoice_id > %s', (floor,))
|
||||||
|
erp_ids = set(int(r[0]) for r in pgc.fetchall())
|
||||||
|
# 3) manquants = à traiter
|
||||||
|
todo = sorted(i for i in f_ids if i not in erp_ids)
|
||||||
|
if LIMIT > 0: todo = todo[:LIMIT]
|
||||||
|
log(" fenêtre id>{} : F={} · ERP={} · à insérer (nouveaux + trous) = {}".format(floor, len(f_ids), len(erp_ids), len(todo)))
|
||||||
|
|
||||||
|
invoices = []
|
||||||
|
items_by_inv = {}
|
||||||
|
for s in range(0, len(todo), 5000):
|
||||||
|
batch = todo[s:s+5000]
|
||||||
|
if not batch: continue
|
||||||
|
ph = ",".join(["%s"]*len(batch))
|
||||||
|
cur.execute("SELECT id, account_id, date_orig, due_date, total_amt, billed_amt, billing_status FROM invoice WHERE id IN ({}) ORDER BY id".format(ph), batch)
|
||||||
|
invoices.extend(cur.fetchall())
|
||||||
|
cur.execute("SELECT ii.*, s.delivery_id AS service_delivery_id FROM invoice_item ii LEFT JOIN service s ON s.id=ii.service_id WHERE ii.invoice_id IN ({}) ORDER BY ii.invoice_id, ii.id".format(ph), batch)
|
||||||
|
for r in cur.fetchall():
|
||||||
|
items_by_inv.setdefault(r["invoice_id"], []).append(r)
|
||||||
|
# mapping SKU→compte produit (product → product_cat.num_compte)
|
||||||
|
cur.execute("SELECT p.sku, pc.num_compte FROM product p JOIN product_cat pc ON p.category=pc.id WHERE p.sku IS NOT NULL AND pc.num_compte IS NOT NULL")
|
||||||
|
sku_to_gl = {}
|
||||||
|
for r in cur.fetchall():
|
||||||
|
an = str(int(r["num_compte"])) if r["num_compte"] else None
|
||||||
|
if an and an in gl_by_number: sku_to_gl[r["sku"]] = gl_by_number[an]
|
||||||
|
mc.close()
|
||||||
|
existing = set() # `todo` exclut déjà les présents ; ON CONFLICT couvre toute course
|
||||||
|
|
||||||
|
by_status = {"Paid": 0, "Unpaid": 0, "Overdue": 0, "Return": 0}
|
||||||
|
ok = skip = no_cust = item_ok = 0
|
||||||
|
samples = []
|
||||||
|
|
||||||
|
for inv in invoices:
|
||||||
|
if inv["id"] in existing: skip += 1; continue
|
||||||
|
cd = cust_map.get(inv["account_id"])
|
||||||
|
if not cd: no_cust += 1; continue
|
||||||
|
cust_name, cust_disp = cd
|
||||||
|
total = r2(inv["total_amt"]); billed = r2(inv["billed_amt"])
|
||||||
|
posting = ts_to_date(inv["date_orig"]) or td
|
||||||
|
due = ts_to_date(inv["due_date"]) or posting
|
||||||
|
is_return, owed, status = map_status(total, billed, due, td)
|
||||||
|
by_status[status] += 1
|
||||||
|
name = sinv_name(inv["id"])
|
||||||
|
if len(samples) < 8:
|
||||||
|
samples.append({"sinv": name, "status": status, "total": total, "outstanding": owed})
|
||||||
|
|
||||||
|
if not APPLY:
|
||||||
|
ok += 1
|
||||||
|
item_ok += len(items_by_inv.get(inv["id"], []))
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
naming_series, customer, customer_name, company,
|
||||||
|
posting_date, due_date, currency, conversion_rate,
|
||||||
|
selling_price_list, price_list_currency,
|
||||||
|
base_grand_total, grand_total, base_net_total, net_total, base_total, total,
|
||||||
|
outstanding_amount, base_rounded_total, rounded_total,
|
||||||
|
is_return, is_debit_note, disable_rounded_total,
|
||||||
|
debit_to, party_account_currency, status, legacy_invoice_id
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,1,0,
|
||||||
|
'ACC-SINV-.YYYY.-',%s,%s,%s,
|
||||||
|
%s,%s,'CAD',1,
|
||||||
|
'Standard Selling','CAD',
|
||||||
|
%s,%s,%s,%s,%s,%s,
|
||||||
|
%s,%s,%s,
|
||||||
|
%s,0,1,
|
||||||
|
%s,'CAD',%s,%s)
|
||||||
|
ON CONFLICT (name) DO NOTHING
|
||||||
|
""", (name, ts, ts, ADMIN, ADMIN,
|
||||||
|
cust_name, cust_disp, COMPANY,
|
||||||
|
posting, due,
|
||||||
|
total, total, total, total, total, total,
|
||||||
|
owed, total, total,
|
||||||
|
1 if is_return else 0,
|
||||||
|
receivable, status, inv["id"]))
|
||||||
|
|
||||||
|
for j, li in enumerate(items_by_inv.get(inv["id"], [])):
|
||||||
|
sku = clean(li.get("sku")) or "MISC"
|
||||||
|
qty = float(li.get("quantity") or 1)
|
||||||
|
rate = float(li.get("unitary_price") or 0)
|
||||||
|
amount = round(qty * rate, 2)
|
||||||
|
desc = (clean(li.get("product_name")) or sku)[:140]
|
||||||
|
item_code = sku if sku in valid_items else None
|
||||||
|
income = sku_to_gl.get(sku, income_default)
|
||||||
|
sloc = loc_by_delivery.get(li.get("service_delivery_id"))
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabSales Invoice Item" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
item_code, item_name, description, qty, rate, amount,
|
||||||
|
base_rate, base_amount, base_net_rate, base_net_amount, net_rate, net_amount,
|
||||||
|
stock_uom, uom, conversion_factor, income_account, cost_center, service_location,
|
||||||
|
parent, parentfield, parenttype
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,1,%s,
|
||||||
|
%s,%s,%s,%s,%s,%s,
|
||||||
|
%s,%s,%s,%s,%s,%s,
|
||||||
|
'Nos','Nos',1,%s,'Main - T',%s,
|
||||||
|
%s,'items','Sales Invoice')
|
||||||
|
ON CONFLICT (name) DO NOTHING
|
||||||
|
""", (uid("SII-"), ts, ts, ADMIN, ADMIN, j+1,
|
||||||
|
item_code, desc, desc, qty, rate, amount,
|
||||||
|
rate, amount, rate, amount, rate, amount,
|
||||||
|
income, sloc, name))
|
||||||
|
item_ok += 1
|
||||||
|
ok += 1
|
||||||
|
except Exception as e:
|
||||||
|
pg.rollback()
|
||||||
|
log(" ERR inv#{} -> {}".format(inv["id"], str(e)[:160]))
|
||||||
|
continue
|
||||||
|
if ok % 2000 == 0:
|
||||||
|
pg.commit(); log(" ... {} insérées".format(ok))
|
||||||
|
|
||||||
|
if APPLY: pg.commit()
|
||||||
|
pg.close()
|
||||||
|
log("")
|
||||||
|
log("="*60)
|
||||||
|
log("Statut cible (reflète F) : {}".format(by_status))
|
||||||
|
log("{} : {} factures, {} lignes, {} déjà présentes, {} sans client".format(
|
||||||
|
"INSÉRÉES" if APPLY else "À INSÉRER (dry-run)", ok, item_ok, skip, no_cust))
|
||||||
|
log("Échantillon : {}".format(samples))
|
||||||
|
log("="*60)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
238
scripts/targo-sync/sync_services_incremental.py
Normal file
238
scripts/targo-sync/sync_services_incremental.py
Normal file
|
|
@ -0,0 +1,238 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Sync INCRÉMENTAL F → ERPNext — couche OPÉRATIONNELLE adresses + services.
|
||||||
|
|
||||||
|
Crée les enregistrements MANQUANTS (diff complet par clé legacy, petites tables → léger) :
|
||||||
|
• delivery → Service Location (LOC-{legacy_delivery_id zfill10}) — mapping migrate_locations.py
|
||||||
|
• service (status=1) → Service Subscription (SUB-{legacy_service_id zfill10}) — mapping import_services_and_enrich_customers.py
|
||||||
|
|
||||||
|
NE crée PAS le doctype standard `Subscription` (moteur de facturation récurrente) : F est
|
||||||
|
autoritaire pour la facturation (scheduler ERPNext en pause) → couche billing gérée à la bascule,
|
||||||
|
comme GL/PLE pour les factures. Ici = miroir OPÉRATIONNEL seulement (ce que les équipes voient).
|
||||||
|
|
||||||
|
Ordre : Service Location d'ABORD (un service requiert sa location + le client). Idempotent (ON CONFLICT).
|
||||||
|
DRY-RUN par défaut ; n'écrit RIEN sans APPLY=1. Options env : LIMIT (lot), PG_HOST.
|
||||||
|
|
||||||
|
Lancer :
|
||||||
|
docker cp sync_services_incremental.py erpnext-backend-1:/tmp/sync_svc.py
|
||||||
|
docker exec -e APPLY=0 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py
|
||||||
|
docker exec -e APPLY=1 -e PG_HOST=db erpnext-backend-1 /home/frappe/frappe-bench/env/bin/python /tmp/sync_svc.py
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import uuid
|
||||||
|
import pymysql
|
||||||
|
import psycopg2
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from html import unescape
|
||||||
|
|
||||||
|
LEGACY = {"host": os.environ.get("LEGACY_HOST", "10.100.80.100"), "user": "facturation",
|
||||||
|
"password": os.environ.get("LEGACY_PW", "VD67owoj"), "database": "gestionclient",
|
||||||
|
"connect_timeout": 30, "read_timeout": 600}
|
||||||
|
PG = {"host": os.environ.get("PG_HOST", "db"), "port": 5432, "user": "postgres",
|
||||||
|
"password": os.environ.get("PG_PW", "123"), "dbname": os.environ.get("PG_DB", "_eb65bdc0c4b1b2d6")}
|
||||||
|
ADMIN = "Administrator"
|
||||||
|
APPLY = os.environ.get("APPLY", "0") == "1"
|
||||||
|
LIMIT = int(os.environ.get("LIMIT", "0"))
|
||||||
|
|
||||||
|
# ── mappings (identiques à la migration) ──
|
||||||
|
PROD_CAT_MAP = {4: "Internet", 32: "Internet", 8: "Internet", 26: "Internet", 29: "Internet",
|
||||||
|
7: "Internet", 23: "Internet", 17: "Internet", 16: "Internet", 21: "Internet",
|
||||||
|
33: "IPTV", 34: "IPTV", 9: "VoIP", 15: "Hébergement", 11: "Hébergement",
|
||||||
|
30: "Hébergement", 10: "Autre", 13: "Autre"}
|
||||||
|
RECUR_MAP = {1: "Mensuel", 2: "Mensuel", 3: "Trimestriel", 4: "Annuel"}
|
||||||
|
|
||||||
|
def now(): return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
def uid(p=""): return p + uuid.uuid4().hex[:10]
|
||||||
|
def clean(v): return unescape(str(v)).strip() if v else ""
|
||||||
|
def loc_name(i): return "LOC-{:010d}".format(int(i))
|
||||||
|
def sub_name(i): return "SUB-{:010d}".format(int(i))
|
||||||
|
def ts_date(t, dflt=None):
|
||||||
|
if not t or int(t) <= 0: return dflt
|
||||||
|
try: return datetime.fromtimestamp(int(t), tz=timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
except Exception: return dflt
|
||||||
|
def log(m): print(m, flush=True)
|
||||||
|
|
||||||
|
def normalize_postal(pc): # "j0s1b0" → "J0S 1B0"
|
||||||
|
s = re.sub(r"\s+", "", clean(pc)).upper()
|
||||||
|
return s[:3] + " " + s[3:6] if len(s) >= 6 else clean(pc)
|
||||||
|
|
||||||
|
def clean_addr(raw, postal=None): # strip postal qui a fui dans address1 + espaces
|
||||||
|
a = clean(raw)
|
||||||
|
if postal:
|
||||||
|
pc = re.sub(r"\s+", "", postal).upper()
|
||||||
|
a = re.sub(r"\s*" + re.escape(pc[:3]) + r"\s*" + re.escape(pc[3:6]) + r"\s*$", "", a, flags=re.I)
|
||||||
|
a = re.sub(r"\s*" + re.escape(pc) + r"\s*$", "", a, flags=re.I)
|
||||||
|
return re.sub(r"\s+", " ", a).strip()
|
||||||
|
|
||||||
|
def svc_status(s, d, resiliated=False): # règle canonique (= legacy-sync.svcStatus) : F service → statut SS
|
||||||
|
if resiliated:
|
||||||
|
return "Annulé" # FIX 2026-07-06 : compte F résilié (status 3/4/5) → l'abo ne peut PAS rester Actif.
|
||||||
|
# F ne cascade PAS la résiliation du compte vers service.status (cf feedback_ghost_active_subscriptions).
|
||||||
|
if int(s or 0) == 1:
|
||||||
|
return "Actif" # CORRIGE 2026-06-13: F status=1 = ACTIF (date_suspended = historique, ignore)
|
||||||
|
return "Annulé"
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ts = now()
|
||||||
|
log("=== Sync incrémental adresses + services (OPÉRATIONNEL, sans Subscription billing) ===")
|
||||||
|
log(" mode = {}".format("APPLY" if APPLY else "DRY-RUN"))
|
||||||
|
pg = psycopg2.connect(**PG); pgc = pg.cursor()
|
||||||
|
mc = pymysql.connect(**LEGACY); cur = mc.cursor(pymysql.cursors.DictCursor)
|
||||||
|
|
||||||
|
# maps communs
|
||||||
|
pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id>0')
|
||||||
|
cust_map = {int(r[0]): r[1] for r in pgc.fetchall()}
|
||||||
|
cur.execute("SELECT id, account_id FROM delivery"); del_acct = {r["id"]: r["account_id"] for r in cur.fetchall()}
|
||||||
|
# FIX 2026-07-06 — comptes résiliés en F (status 3/4/5) : leurs services restent souvent status=1
|
||||||
|
# (F ne cascade pas) → sans ce garde-fou, Phase B/D les (re)mettent 'Actif' = abonnements fantômes.
|
||||||
|
cur.execute("SELECT id FROM account WHERE status IN (3,4,5)")
|
||||||
|
resil_acct = set(r["id"] for r in cur.fetchall())
|
||||||
|
log(" Comptes F résiliés (status 3/4/5) : {}".format(len(resil_acct)))
|
||||||
|
|
||||||
|
# ═══ PHASE A — Service Location (delivery) ═══
|
||||||
|
cur.execute("SELECT id FROM delivery")
|
||||||
|
f_del = [r["id"] for r in cur.fetchall()]
|
||||||
|
pgc.execute('SELECT legacy_delivery_id FROM "tabService Location" WHERE legacy_delivery_id>0')
|
||||||
|
erp_loc = set(int(r[0]) for r in pgc.fetchall())
|
||||||
|
miss_loc = sorted(i for i in f_del if i not in erp_loc)
|
||||||
|
if LIMIT: miss_loc = miss_loc[:LIMIT]
|
||||||
|
log(" Adresses : F={} ERP={} → {} à créer".format(len(f_del), len(erp_loc), len(miss_loc)))
|
||||||
|
loc_ok = loc_skip = 0; loc_samples = []
|
||||||
|
if miss_loc:
|
||||||
|
for s in range(0, len(miss_loc), 2000):
|
||||||
|
batch = miss_loc[s:s+2000]
|
||||||
|
cur.execute("SELECT id, account_id, name, address1, city, state, zip FROM delivery WHERE id IN ({})".format(",".join(["%s"]*len(batch))), batch)
|
||||||
|
for d in cur.fetchall():
|
||||||
|
postal = normalize_postal(d.get("zip"))
|
||||||
|
addr = clean_addr(d.get("address1"), postal)
|
||||||
|
city = clean_addr(d.get("city"))
|
||||||
|
lname = (clean(d.get("name")) or ("{}, {}".format(addr, city) if addr else "Location-{}".format(d["id"])))[:140]
|
||||||
|
cust = cust_map.get(d["account_id"])
|
||||||
|
if len(loc_samples) < 6: loc_samples.append({"loc": loc_name(d["id"]), "addr": addr, "city": city, "customer": cust})
|
||||||
|
if not APPLY: loc_ok += 1; continue
|
||||||
|
try:
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabService Location" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
customer, location_name, status, address_line, city, postal_code, province, legacy_delivery_id
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,0,0, %s,%s,'Active',%s,%s,%s,%s,%s)
|
||||||
|
ON CONFLICT (name) DO NOTHING
|
||||||
|
""", (loc_name(d["id"]), ts, ts, ADMIN, ADMIN, cust, lname, addr, city, postal, clean(d.get("state")), d["id"]))
|
||||||
|
loc_ok += 1
|
||||||
|
except Exception as e:
|
||||||
|
loc_skip += 1; pg.rollback()
|
||||||
|
if loc_skip <= 5: log(" ERR loc#{} -> {}".format(d["id"], str(e)[:120]))
|
||||||
|
continue
|
||||||
|
if APPLY: pg.commit()
|
||||||
|
# recharge le map location pour la phase B
|
||||||
|
pgc.execute('SELECT legacy_delivery_id, name FROM "tabService Location" WHERE legacy_delivery_id>0')
|
||||||
|
loc_map = {int(r[0]): r[1] for r in pgc.fetchall()}
|
||||||
|
|
||||||
|
# ═══ PHASE B — Service Subscription (service status=1) ═══
|
||||||
|
cur.execute("SELECT id FROM service WHERE status=1")
|
||||||
|
f_svc = [r["id"] for r in cur.fetchall()]
|
||||||
|
pgc.execute('SELECT legacy_service_id FROM "tabService Subscription" WHERE legacy_service_id>0')
|
||||||
|
erp_sub = set(int(r[0]) for r in pgc.fetchall())
|
||||||
|
miss_svc = sorted(i for i in f_svc if i not in erp_sub)
|
||||||
|
if LIMIT: miss_svc = miss_svc[:LIMIT]
|
||||||
|
log(" Services : F actifs={} ERP={} → {} à créer".format(len(f_svc), len(erp_sub), len(miss_svc)))
|
||||||
|
# product names FR
|
||||||
|
cur.execute("SELECT product_id, name AS prod_name FROM product_translate WHERE language_id='francais'")
|
||||||
|
prod_names = {r["product_id"]: r["prod_name"] for r in cur.fetchall()}
|
||||||
|
sub_ok = sub_noloc = sub_skip = 0; sub_samples = []
|
||||||
|
if miss_svc:
|
||||||
|
for s in range(0, len(miss_svc), 2000):
|
||||||
|
batch = miss_svc[s:s+2000]
|
||||||
|
cur.execute("""SELECT s.id, s.delivery_id, s.product_id, s.payment_recurrence, s.hijack, s.hijack_price,
|
||||||
|
s.hijack_download_speed, s.hijack_upload_speed, s.date_orig, s.date_end_contract,
|
||||||
|
s.radius_user, s.radius_pwd,
|
||||||
|
p.sku, p.price, p.download_speed, p.upload_speed, p.category AS prod_cat
|
||||||
|
FROM service s LEFT JOIN product p ON s.product_id=p.id
|
||||||
|
WHERE s.id IN ({})""".format(",".join(["%s"]*len(batch))), batch)
|
||||||
|
for svc in cur.fetchall():
|
||||||
|
sl = loc_map.get(svc["delivery_id"]) if svc["delivery_id"] else None
|
||||||
|
acct_id = del_acct.get(svc["delivery_id"]) if svc["delivery_id"] else None
|
||||||
|
cust = cust_map.get(acct_id) if acct_id else None
|
||||||
|
if not sl or not cust: sub_noloc += 1; continue # location + client requis (idem migration)
|
||||||
|
new_status = "Annulé" if acct_id in resil_acct else "Actif" # FIX 2026-07-06 : jamais Actif sur compte résilié
|
||||||
|
cat = PROD_CAT_MAP.get(svc["prod_cat"] or 0, "Autre")
|
||||||
|
plan = unescape(prod_names.get(svc["product_id"], svc.get("sku") or "Unknown"))[:140]
|
||||||
|
if svc["hijack"] and svc["hijack_download_speed"]:
|
||||||
|
sd, su = int(svc["hijack_download_speed"]) // 1024, int(svc["hijack_upload_speed"] or 0) // 1024
|
||||||
|
elif svc["download_speed"]:
|
||||||
|
sd, su = int(svc["download_speed"]) // 1024, int(svc["upload_speed"] or 0) // 1024
|
||||||
|
else: sd = su = 0
|
||||||
|
price = float(svc["hijack_price"] or 0) if svc["hijack"] else float(svc["price"] or 0)
|
||||||
|
cycle = RECUR_MAP.get(svc["payment_recurrence"], "Mensuel")
|
||||||
|
start = ts_date(svc["date_orig"], "2020-01-01"); end = ts_date(svc["date_end_contract"])
|
||||||
|
if len(sub_samples) < 6: sub_samples.append({"sub": sub_name(svc["id"]), "cat": cat, "plan": plan, "price": price, "cycle": cycle, "loc": sl})
|
||||||
|
if not APPLY: sub_ok += 1; continue
|
||||||
|
try:
|
||||||
|
pgc.execute("""
|
||||||
|
INSERT INTO "tabService Subscription" (
|
||||||
|
name, creation, modified, modified_by, owner, docstatus, idx,
|
||||||
|
customer, service_location, status, service_category, plan_name,
|
||||||
|
speed_down, speed_up, monthly_price, billing_cycle, start_date, end_date,
|
||||||
|
legacy_service_id, radius_user, radius_password, product_sku
|
||||||
|
) VALUES (%s,%s,%s,%s,%s,0,0, %s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s, %s,%s,%s,%s)
|
||||||
|
ON CONFLICT (name) DO NOTHING
|
||||||
|
""", (sub_name(svc["id"]), ts, ts, ADMIN, ADMIN, cust, sl, new_status, cat, plan,
|
||||||
|
sd, su, price, cycle, start, end,
|
||||||
|
svc["id"], clean(svc.get("radius_user")), clean(svc.get("radius_pwd")), clean(svc.get("sku"))))
|
||||||
|
sub_ok += 1
|
||||||
|
except Exception as e:
|
||||||
|
sub_skip += 1; pg.rollback()
|
||||||
|
if sub_skip <= 5: log(" ERR svc#{} -> {}".format(svc["id"], str(e)[:120]))
|
||||||
|
continue
|
||||||
|
if APPLY: pg.commit()
|
||||||
|
|
||||||
|
# ═══ PHASE C — rafraîchir le PRIX des SS existantes (F autoritaire ; opérationnel) ═══
|
||||||
|
# Les prix F (rabais/crédits/forfaits) bougent après la migration → le monthly_price figé
|
||||||
|
# de la SS diverge. On le réaligne sur F. (Statut Actif/Suspendu/Annulé = legacy-sync.syncServices.)
|
||||||
|
cur.execute("SELECT s.id, IF(s.hijack=1, COALESCE(s.hijack_price,0), COALESCE(p.price,0)) AS fprice FROM service s LEFT JOIN product p ON p.id=s.product_id WHERE s.status=1")
|
||||||
|
fprice = {r["id"]: round(float(r["fprice"]), 2) for r in cur.fetchall()}
|
||||||
|
pgc.execute('SELECT legacy_service_id, monthly_price FROM "tabService Subscription" WHERE legacy_service_id>0')
|
||||||
|
to_upd = []
|
||||||
|
for lid, mp in pgc.fetchall():
|
||||||
|
f = fprice.get(int(lid))
|
||||||
|
if f is None: continue
|
||||||
|
if abs(round(float(mp or 0), 2) - f) > 0.011: to_upd.append((f, int(lid)))
|
||||||
|
price_fixed = 0
|
||||||
|
log(" Prix SS divergents vs F : {}".format(len(to_upd)))
|
||||||
|
if APPLY and to_upd:
|
||||||
|
for f, lid in to_upd:
|
||||||
|
pgc.execute('UPDATE "tabService Subscription" SET monthly_price=%s, modified=%s WHERE legacy_service_id=%s AND ROUND(monthly_price::numeric,2)<>%s', (f, ts, lid, f))
|
||||||
|
pg.commit(); price_fixed = len(to_upd)
|
||||||
|
|
||||||
|
# ═══ PHASE D — rafraîchir le STATUT des SS (Actif/Suspendu/Annulé) depuis F ═══
|
||||||
|
# Reflète résiliations (status 1→0 = Annulé) + suspensions (date_suspended>0 = Suspendu) + réactivations.
|
||||||
|
# C'est la SOURCE du statut client (disabled dérivé des services — cf. feedback_customer_status_policy).
|
||||||
|
cur.execute("SELECT s.id, s.status, s.date_suspended, d.account_id FROM service s LEFT JOIN delivery d ON d.id=s.delivery_id")
|
||||||
|
want = {r["id"]: svc_status(r["status"], r["date_suspended"], r["account_id"] in resil_acct) for r in cur.fetchall()}
|
||||||
|
pgc.execute('SELECT legacy_service_id, status FROM "tabService Subscription" WHERE legacy_service_id>0')
|
||||||
|
st_upd = []; trans = {}
|
||||||
|
for lid, st in pgc.fetchall():
|
||||||
|
w = want.get(int(lid))
|
||||||
|
if w and w != st:
|
||||||
|
st_upd.append((w, int(lid))); k = (st or "∅") + "→" + w; trans[k] = trans.get(k, 0) + 1
|
||||||
|
status_fixed = 0
|
||||||
|
log(" Statut SS divergents vs F : {} {}".format(len(st_upd), trans))
|
||||||
|
if APPLY and st_upd:
|
||||||
|
for w, lid in st_upd:
|
||||||
|
pgc.execute('UPDATE "tabService Subscription" SET status=%s, modified=%s WHERE legacy_service_id=%s AND status<>%s', (w, ts, lid, w))
|
||||||
|
pg.commit(); status_fixed = len(st_upd)
|
||||||
|
|
||||||
|
mc.close(); pg.close()
|
||||||
|
log("")
|
||||||
|
log("=" * 60)
|
||||||
|
log("Statut SS réalignés sur F : {} {}".format(status_fixed if APPLY else len(st_upd), "corrigés" if APPLY else "à corriger (dry-run)"))
|
||||||
|
log("Prix SS réalignés sur F : {} {}".format(price_fixed if APPLY else len(to_upd), "corrigés" if APPLY else "à corriger (dry-run)"))
|
||||||
|
log("Adresses : {} {} | {} err".format(loc_ok, "créées" if APPLY else "à créer (dry-run)", loc_skip))
|
||||||
|
log(" échantillon: {}".format(loc_samples))
|
||||||
|
log("Services : {} {} | {} sans location/client | {} err".format(sub_ok, "créés" if APPLY else "à créer (dry-run)", sub_noloc, sub_skip))
|
||||||
|
log(" échantillon: {}".format(sub_samples))
|
||||||
|
log("=" * 60)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -501,6 +501,78 @@ async function syncServices ({ confirm = false } = {}) {
|
||||||
return { ...plan, applied: ok, errors: err, error_samples: errs }
|
return { ...plan, applied: ok, errors: err, error_samples: errs }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── CORRECTION : abonnements « fantômes » sur comptes F RÉSILIÉS ─────────────
|
||||||
|
// Bug : F résilie au niveau COMPTE (account.status 3/4/5) mais laisse service.status=1 → le
|
||||||
|
// mapping svcStatus (basé sur service.status seul) garde l'abonnement 'Actif'. Ici on force
|
||||||
|
// 'Annulé' pour tout abonnement 'Actif' dont le client a un legacy_account_id RÉSILIÉ.
|
||||||
|
// Robuste (indépendant du bridge : fResiliatedAccountIds + jointure PG). Idempotent → réexécutable
|
||||||
|
// périodiquement (fix « forward »). dry-run sauf confirm==='F-WINS'. Cf feedback_ghost_active_subscriptions.
|
||||||
|
async function annulResiliatedSubscriptions ({ confirm = false, limit = 20000 } = {}) {
|
||||||
|
const resil = await fResiliatedAccountIds()
|
||||||
|
if (!resil || !resil.length) return { resiliated_accounts: 0, would_update: 0 }
|
||||||
|
const pg = require('./address-db').pool()
|
||||||
|
const rows = []
|
||||||
|
for (let i = 0; i < resil.length; i += 2000) {
|
||||||
|
const b = resil.slice(i, i + 2000)
|
||||||
|
const r = await pg.query('SELECT ss.name, ss.customer, ss.monthly_price FROM "tabService Subscription" ss JOIN "tabCustomer" c ON ss.customer=c.name WHERE ss.status=$1 AND c.legacy_account_id = ANY($2)', ['Actif', b])
|
||||||
|
rows.push(...r.rows)
|
||||||
|
}
|
||||||
|
const plan = { resiliated_accounts: resil.length, would_update: rows.length, monthly_sum: Math.round(rows.reduce((s, x) => s + (Number(x.monthly_price) || 0), 0)) }
|
||||||
|
if (confirm !== 'F-WINS') return { dry_run: true, ...plan, sample: rows.slice(0, 15).map(x => ({ name: x.name, customer: x.customer })) }
|
||||||
|
let ok = 0; let err = 0; const errs = []; let i = 0
|
||||||
|
for (const x of rows.slice(0, limit)) {
|
||||||
|
try { const r = await withRetry(() => erp.update('Service Subscription', x.name, { status: 'Annulé' }), 'annul ' + x.name); if (r && r.ok !== false) ok++; else { err++; if (errs.length < 15) errs.push({ name: x.name, err: (r && r.error) || 'fail' }) } }
|
||||||
|
catch (e) { err++; if (errs.length < 15) errs.push({ name: x.name, err: e.message }) }
|
||||||
|
if (++i % 25 === 0) await new Promise(r => setTimeout(r, 200)) // throttle
|
||||||
|
}
|
||||||
|
log(`legacy-sync annulResiliatedSubscriptions: ${ok} annulés, ${err} err`)
|
||||||
|
return { ...plan, applied: ok, errors: err, error_samples: errs }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tableau de bord de réconciliation : F (mysql) vs ERPNext (pg), par entité ──
|
||||||
|
// Chaque compte est indépendant (try/catch → null si table absente) pour ne jamais casser la vue.
|
||||||
|
async function scoreboard () {
|
||||||
|
const pg = require('./address-db').pool()
|
||||||
|
const pgc = async (sql, params) => { try { return Number((await pg.query(sql, params)).rows[0].c) } catch (e) { return null } }
|
||||||
|
const mp = pool()
|
||||||
|
const myc = async (sql) => { if (!mp) return null; try { const [r] = await mp.query({ sql, timeout: 8000 }); return Number(r[0].c) } catch (e) { return null } }
|
||||||
|
let resil = []; try { resil = await fResiliatedAccountIds() } catch (e) {}
|
||||||
|
const [fAcct, fSvcActive, fDelivery, fTicket, fInvoice, fPayment, fDevice,
|
||||||
|
eCust, eSubActif, eLoc, eIssue, eInv, ePay, eDevice, ghost] = await Promise.all([
|
||||||
|
myc('SELECT COUNT(*) c FROM account'),
|
||||||
|
// F actif = service.status=1 SUR compte non-résilié (F ne cascade pas la résiliation → comptage brut gonflé) : comparé pomme-à-pomme au 'Actif' ERPNext maintenant nettoyé
|
||||||
|
myc('SELECT COUNT(*) c FROM service s JOIN delivery d ON d.id=s.delivery_id JOIN account a ON a.id=d.account_id WHERE s.status=1 AND a.status NOT IN (3,4,5)'),
|
||||||
|
myc('SELECT COUNT(*) c FROM delivery'),
|
||||||
|
myc('SELECT COUNT(*) c FROM ticket'),
|
||||||
|
myc('SELECT COUNT(*) c FROM invoice'),
|
||||||
|
myc('SELECT COUNT(*) c FROM payment'),
|
||||||
|
myc('SELECT COUNT(*) c FROM device'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabCustomer"'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabService Subscription" WHERE status=$1', ['Actif']),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabService Location"'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabIssue"'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabSales Invoice"'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabPayment Entry"'),
|
||||||
|
pgc('SELECT COUNT(*) c FROM "tabService Equipment"'),
|
||||||
|
resil.length ? pgc('SELECT COUNT(*) c FROM "tabService Subscription" ss JOIN "tabCustomer" c ON ss.customer=c.name WHERE ss.status=$1 AND c.legacy_account_id = ANY($2)', ['Actif', resil]) : 0,
|
||||||
|
])
|
||||||
|
const row = (key, label, f, erp, note) => ({ key, label, f, erp, delta: (f != null && erp != null) ? f - erp : null, note: note || '' })
|
||||||
|
return {
|
||||||
|
generated_at: new Date().toISOString(),
|
||||||
|
resiliated_f_accounts: resil.length,
|
||||||
|
ghost_active_subscriptions: ghost,
|
||||||
|
entities: [
|
||||||
|
row('customers', 'Clients', fAcct, eCust, 'F account ↔ Customer'),
|
||||||
|
row('locations', 'Lieux de service', fDelivery, eLoc, 'F delivery ↔ Service Location'),
|
||||||
|
row('subscriptions', 'Abonnements actifs', fSvcActive, eSubActif, ghost ? `${ghost} fantômes (compte résilié) à annuler` : 'F actif hors compte résilié ↔ Actif'),
|
||||||
|
row('devices', 'Appareils', fDevice, eDevice, 'F device ↔ Service Equipment (liens orphelins possibles)'),
|
||||||
|
row('tickets', 'Tickets', fTicket, eIssue, 'F ticket ↔ Issue (import à réparer)'),
|
||||||
|
row('invoices', 'Factures', fInvoice, eInv, 'F invoice ↔ Sales Invoice'),
|
||||||
|
row('payments', 'Paiements', fPayment, ePay, 'F payment ↔ Payment Entry'),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── CRÉATEUR DE CLIENTS CANONIQUE (utilisé par billing + orchestrateur) ──────
|
// ── CRÉATEUR DE CLIENTS CANONIQUE (utilisé par billing + orchestrateur) ──────
|
||||||
// Crée les Customers ERPNext manquants depuis F (par legacy_account_id) via erp.create
|
// Crée les Customers ERPNext manquants depuis F (par legacy_account_id) via erp.create
|
||||||
// (REST → naming/validations/defaults gérés). Idempotent. POLITIQUE : créé VISIBLE
|
// (REST → naming/validations/defaults gérés). Idempotent. POLITIQUE : créé VISIBLE
|
||||||
|
|
@ -734,6 +806,7 @@ async function handle (req, res, method, p, url) {
|
||||||
try { return json(res, 200, await bridgeCall({ action: 'ping' })) }
|
try { return json(res, 200, await bridgeCall({ action: 'ping' })) }
|
||||||
catch (e) { return json(res, 502, { error: e.message }) }
|
catch (e) { return json(res, 502, { error: e.message }) }
|
||||||
}
|
}
|
||||||
|
if (p === '/legacy-sync/scoreboard' && method === 'GET') { try { return json(res, 200, await scoreboard()) } catch (e) { return json(res, 500, { error: e.message }) } }
|
||||||
if (p === '/legacy-sync/state' && method === 'GET') return json(res, 200, loadState())
|
if (p === '/legacy-sync/state' && method === 'GET') return json(res, 200, loadState())
|
||||||
if (p === '/legacy-sync/run-status' && method === 'GET') { try { return json(res, 200, JSON.parse(fs.readFileSync(RUN_LOG, 'utf8'))) } catch { return json(res, 200, { status: 'idle' }) } }
|
if (p === '/legacy-sync/run-status' && method === 'GET') { try { return json(res, 200, JSON.parse(fs.readFileSync(RUN_LOG, 'utf8'))) } catch { return json(res, 200, { status: 'idle' }) } }
|
||||||
// Liste COMPLÈTE des erreurs du dernier apply, groupées par raison. ?reason=email_invalide pour filtrer.
|
// Liste COMPLÈTE des erreurs du dernier apply, groupées par raison. ?reason=email_invalide pour filtrer.
|
||||||
|
|
@ -748,4 +821,4 @@ async function handle (req, res, method, p, url) {
|
||||||
return json(res, 404, { error: 'route legacy-sync inconnue' })
|
return json(res, 404, { error: 'route legacy-sync inconnue' })
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { handle, runPreview, previewCustomers, applySafeAdds, syncCustomers, syncServices, createCustomers, createCustomersByIds, applyAccountChange, applyReviewBatch, fCounts, domainCounts, loadState, mapAccount, svcStatus, fAccountStatuses, fResiliatedAccountIds, F_ACCT_STATUS_LABEL, GROUP_MAP, CMP }
|
module.exports = { handle, runPreview, previewCustomers, applySafeAdds, syncCustomers, syncServices, annulResiliatedSubscriptions, scoreboard, createCustomers, createCustomersByIds, applyAccountChange, applyReviewBatch, fCounts, domainCounts, loadState, mapAccount, svcStatus, fAccountStatuses, fResiliatedAccountIds, F_ACCT_STATUS_LABEL, GROUP_MAP, CMP }
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user