diff --git a/docs/ARCHITECTURE-COMPARE.md b/docs/ARCHITECTURE-COMPARE.md new file mode 100644 index 0000000..f324395 --- /dev/null +++ b/docs/ARCHITECTURE-COMPARE.md @@ -0,0 +1,385 @@ +# Architecture comparative — Legacy vs ERPNext + +## 1. Schéma relationnel Legacy (gestionclient / MariaDB) + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ CLIENTS & ADRESSES │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ account (15,303) delivery (17,114) │ +│ ├─ id (PK) ├─ id (PK) │ +│ ├─ customer_id (ex: DR2) ├─ account_id → account.id │ +│ ├─ first_name, last_name ├─ name (label) │ +│ ├─ company ├─ address1, city, zip │ +│ ├─ email, tel_home, cell ├─ longitude, latitude │ +│ ├─ address1, city, zip └─ contact, email │ +│ ├─ status (1=actif,4=terminé) │ +│ ├─ group_id (5=rés, 8=com) │ +│ ├─ ppa, ppa_code, ppa_account ← PPA AccesD Desjardins │ +│ ├─ stripe_id ← Paiement carte │ +│ ├─ tax_group │ +│ └─ language_id │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ SERVICES & FORFAITS │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ service (66,879) product (833) │ +│ ├─ id (PK) ├─ id (PK) │ +│ ├─ delivery_id → delivery.id ├─ sku (ex: FTTH150I) │ +│ ├─ device_id → device.id ├─ price │ +│ ├─ product_id → product.id ├─ active │ +│ ├─ status (1=actif, 0=inactif) ├─ category → product_cat.id │ +│ ├─ date_orig, date_next_invoice ├─ download_speed (Kbps) │ +│ ├─ payment_recurrence ├─ upload_speed (Kbps) │ +│ ├─ hijack (prix spécial) ├─ quota_day, quota_night │ +│ ├─ hijack_price, hijack_desc ├─ fibre_lineprofile │ +│ ├─ radius_user, radius_pwd └─ fibre_serviceprofile │ +│ ├─ date_end_contract │ +│ └─ actif_until product_cat (34) │ +│ ├─ id, name │ +│ Note: service EST l'abonnement. ├─ num_compte (→ compta) │ +│ Un client a N services actifs. └─ combo_dispo │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ FACTURATION & PAIEMENTS │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ invoice (629,944) payment (540,522) │ +│ ├─ id (PK) ├─ id (PK) │ +│ ├─ account_id → account.id ├─ account_id → account.id │ +│ ├─ date_orig (unix timestamp) ├─ date_orig │ +│ ├─ total_amt, billed_amt ├─ amount │ +│ ├─ billing_status (0=draft,1=ok) ├─ type (chèque,PPA,CC...) │ +│ ├─ process_status ├─ reference │ +│ ├─ due_date └─ applied_amt │ +│ ├─ notes, template_message │ +│ └─ ppa_charge payment_item (684,778) │ +│ ├─ payment_id → payment.id │ +│ invoice_item (1,859,260) ├─ invoice_id → invoice.id │ +│ ├─ invoice_id → invoice.id └─ amount │ +│ ├─ service_id → service.id │ +│ ├─ sku → product.sku invoice_tax (par facture) │ +│ ├─ quantity, unitary_price ├─ invoice_id │ +│ └─ product_name ├─ tax_id → tax.id │ +│ └─ amount │ +│ tax (4) │ +│ ├─ TPS 5%, TVQ 8.925/9.975% │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ COMPTABILITÉ │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ compta_comptes (48) compta_journal_ecriture (1.2M)│ +│ ├─ id, num_compte (ex: 4020) ├─ id │ +│ ├─ category (Actif/Passif/Rev) ├─ date │ +│ └─ desc ├─ compta_id → comptes.id │ +│ ├─ debit, credit │ +│ compta_setup └─ desc │ +│ └─ month_closed (fermeture) │ +│ │ +│ LOGIQUE: product_cat.num_compte lie chaque catégorie │ +│ de produit à un compte de revenus (ex: cat 32 → 4020) │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ RÉSEAU & ÉQUIPEMENTS │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ device (10,377) fibre (16,057) │ +│ ├─ id (PK) ├─ id (PK) │ +│ ├─ delivery_id → delivery.id ├─ service_id → service.id │ +│ ├─ category, name ├─ terrain, rue, ville │ +│ ├─ sn (serial number) ├─ frame, slot, port │ +│ ├─ mac ├─ vlan_manage/internet/tel │ +│ ├─ model, manufacturier ├─ ontid, sn │ +│ ├─ manage (IP admin) ├─ info_connect (IP OLT) │ +│ ├─ user, pass ├─ latitude, longitude │ +│ └─ parent (device hiérarchie) └─ placemarks_id │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ TICKETS & SUPPORT │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ticket (242,618) ticket_msg (784,290) │ +│ ├─ id (PK) ├─ id │ +│ ├─ account_id → account.id ├─ ticket_id → ticket.id │ +│ ├─ delivery_id → delivery.id ├─ msg (mediumtext, base64) │ +│ ├─ subject └─ unread_csv │ +│ ├─ dept_id → ticket_dept.id │ +│ ├─ assign_to → staff.id ticket_dept (21) │ +│ ├─ status (open/pending/closed) ├─ Support, Facturation │ +│ ├─ due_date, priority ├─ Installation, Réparation │ +│ ├─ wizard (JSON install data) └─ Vente, SysAdmin, etc. │ +│ └─ wizard_fibre │ +│ │ +│ bon_travail (14,472) staff (155) │ +│ ├─ account_id, tech1, tech2 ├─ username, password │ +│ ├─ heure_arrive/depart ├─ first_name, last_name │ +│ └─ subtotal, tps, tvq, total ├─ email, cell, ext │ +│ ├─ rights (serialized PHP) │ +│ └─ dept_list │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 2. Schéma ERPNext cible + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ CLIENTS & ADRESSES │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Customer Address │ +│ ├─ name (PK, auto) ├─ name (PK, auto) │ +│ ├─ customer_name ├─ address_title │ +│ ├─ customer_type (Ind/Company) ├─ address_line1, city │ +│ ├─ customer_group ├─ state, pincode, country │ +│ ├─ territory (Canada) ├─ latitude, longitude │ +│ ├─ *legacy_account_id ← NEW ├─ links[] → Customer │ +│ ├─ *legacy_customer_id ← NEW └─ address_type │ +│ ├─ *ppa_enabled ← NEW │ +│ ├─ *stripe_id ← NEW Contact │ +│ └─ disabled ├─ first_name, last_name │ +│ ├─ email_ids[], phone_nos[] │ +│ Note: Customer est SÉPARÉ de └─ links[] → Customer │ +│ Address et Contact (pattern ERPNext) │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ SERVICES & FORFAITS │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Item (type=Service) Subscription │ +│ ├─ item_code (= legacy SKU) ├─ party_type = Customer │ +│ ├─ item_name ├─ party → Customer │ +│ ├─ item_group → Item Group ├─ plans[] → Subscription Plan│ +│ ├─ standard_rate ├─ start_date, end_date │ +│ ├─ *legacy_product_id ← NEW ├─ status (Active/Cancelled) │ +│ ├─ *download_speed ← NEW ├─ generate_invoice_at │ +│ ├─ *upload_speed ← NEW ├─ sales_tax_template │ +│ ├─ *quota_day_gb ← NEW ├─ *radius_user ← NEW │ +│ ├─ *quota_night_gb ← NEW ├─ *radius_pwd ← NEW │ +│ ├─ *fibre_lineprofile ← NEW └─ *legacy_service_id ← NEW │ +│ └─ *fibre_serviceprofile ← NEW │ +│ Subscription Plan │ +│ Item Group (34) ├─ plan_name │ +│ ├─ Services/ ├─ item → Item │ +│ │ ├─ Mensualités fibre ├─ cost (prix fixe) │ +│ │ ├─ Mensualités sans fil ├─ billing_interval = Month │ +│ │ ├─ Téléphonie └─ currency = CAD │ +│ │ └─ ...16 sous-groupes │ +│ ├─ Products/ │ +│ │ ├─ Équipement fibre Note: hijack_price du legacy │ +│ │ ├─ Quincaillerie → additional_discount sur │ +│ │ └─ ...8 sous-groupes la Subscription │ +│ └─ Frais et ajustements/ │ +│ └─ ...10 sous-groupes │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ FACTURATION & PAIEMENTS │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Sales Invoice Payment Entry │ +│ ├─ customer → Customer ├─ party → Customer │ +│ ├─ posting_date ├─ posting_date │ +│ ├─ items[] (child table) ├─ paid_amount │ +│ │ ├─ item_code → Item ├─ references[] (child) │ +│ │ ├─ qty, rate │ ├─ reference_name → SINV │ +│ │ └─ amount │ └─ allocated_amount │ +│ ├─ taxes[] → Tax Template ├─ mode_of_payment │ +│ ├─ grand_total └─ reference_no │ +│ ├─ status (Draft/Submitted/Paid) │ +│ └─ is_return (pour crédits) Payment Reconciliation │ +│ └─ auto-match paiements │ +│ Sales Taxes Template │ +│ ├─ TPS 5% Journal Entry │ +│ └─ TVQ 9.975% └─ Opening balance │ +│ │ +│ NATIF: Subscription génère auto les Sales Invoice chaque mois │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ RÉSEAU & ÉQUIPEMENTS (FSM) │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Service Location (custom) Service Equipment (custom) │ +│ ├─ name (LOC-#####) ├─ name (EQP-#####) │ +│ ├─ customer → Customer ├─ location → Service Loc │ +│ ├─ address → Address ├─ serial_number (SN) │ +│ ├─ gps_lat, gps_lon ├─ mac_address │ +│ ├─ connection_type (FTTH/Wireless) ├─ equipment_type │ +│ ├─ olt_ip, olt_frame/slot/port ├─ status │ +│ └─ vlan_internet/tele/manage └─ model, manufacturer │ +│ │ +│ Dispatch Job (custom) Dispatch Technician (custom) │ +│ ├─ customer, service_location ├─ full_name, phone │ +│ ├─ job_type (Install/Repair/...) ├─ traccar_device_id │ +│ ├─ assigned_tech └─ status │ +│ ├─ scheduled_date/time │ +│ └─ equipment[], checklist[] │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────────┐ +│ TICKETS & SUPPORT │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ Issue (natif ERPNext HD) Employee (natif ERPNext HR) │ +│ ├─ subject ├─ employee_name │ +│ ├─ customer → Customer ├─ department │ +│ ├─ issue_type → ticket_dept ├─ user_id → User │ +│ ├─ priority └─ cell_phone, email │ +│ ├─ status (Open/Replied/Closed) │ +│ ├─ description (HTML) Note: staff.rights (PHP) │ +│ └─ resolution_details → Rôles ERPNext │ +│ │ +│ Issue peut contenir des Comments (Communication) │ +│ → ticket_msg.msg migré ici (base64 images → fichiers) │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 3. Mapping table par table + +| Legacy Table | Records | ERPNext DocType | Custom Fields | Notes | +|---|---|---|---|---| +| account | 15,303 | **Customer** | legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id | status 1→enabled, 4→disabled | +| account (contact) | — | **Contact** | — | email, tel, cell séparés | +| delivery | 17,114 | **Address** + **Service Location** | latitude, longitude | GPS dans Address natif | +| product | 833 | **Item** | legacy_product_id, download/upload_speed, quota_gb, fibre_profiles | ✅ FAIT | +| product_cat | 34 | **Item Group** | — | ✅ FAIT | +| service | 66,879 | **Subscription** | radius_user, radius_pwd, legacy_service_id | hijack → discount | +| invoice | 629,944 | **Sales Invoice** | — | Opening balance pour historique | +| invoice_item | 1,859,260 | **Sales Invoice Item** | — | Idem | +| payment | 540,522 | **Payment Entry** | — | Opening balance pour historique | +| payment_item | 684,778 | **Payment Entry Reference** | — | Idem | +| tax | 4 | **Sales Taxes Template** | — | TPS 5% + TVQ 9.975% | +| compta_comptes | 48 | **Account** (Chart) | — | Map → plan comptable canadien existant | +| compta_journal | 1.2M | **Journal Entry** | — | Opening balance seulement | +| device | 10,377 | **Service Equipment** | — | FSM custom doctype | +| fibre | 16,057 | **Service Location** (champs OLT) | — | FSM custom doctype | +| ticket | 242,618 | **Issue** | — | AI knowledge base | +| ticket_msg | 784,290 | **Communication** | — | base64 → File uploads | +| bon_travail | 14,472 | **Dispatch Job** | — | FSM custom doctype | +| staff | 155 | **Employee** | — | rights → Roles | +| ticket_dept | 21 | **Issue Type** | — | Support, Facturation, etc. | + +--- + +## 4. Différences d'architecture clés + +### Ce qui change fondamentalement + +| Aspect | Legacy | ERPNext | +|---|---|---| +| **Nommage** | ID numérique auto-inc | Naming series (CUST-.YYYY.-, etc.) | +| **Client/Adresse** | Tout dans `account` | Séparé : Customer + Address + Contact | +| **Abonnement** | `service` = ligne active | `Subscription` génère auto les factures | +| **Prix spécial** | `hijack_price` sur service | `additional_discount` sur Subscription ou Pricing Rule | +| **Taxes** | Calculées dans PHP | Template TPS+TVQ auto-appliqué | +| **Comptabilité** | Écritures manuelles PHP | GL Entry auto depuis Sales Invoice | +| **Permissions** | `staff.rights` (PHP serialize) | Rôles + DocType permissions | +| **Fichiers** | Base64 dans mediumtext | File doctype → /files/ directory | +| **RADIUS** | Champs sur `service` | Custom fields sur `Subscription` | +| **Dates** | Unix timestamp (bigint) | ISO date (YYYY-MM-DD) | + +### Ce qui reste pareil +- SKU des produits +- Structure client → N adresses → N services +- Facturation mensuelle récurrente +- Système de tickets/support + +--- + +## 5. Flux opérationnels comparés + +### Facturation mensuelle + +**Legacy :** +``` +CRON → task_charge_recurrent.php + → SELECT services WHERE date_next_invoice < NOW() + → INSERT invoice + invoice_items + → UPDATE service.date_next_invoice + 1 month + → task_generate_statement.php (PDF) + → task_mail.php (envoi email) +``` + +**ERPNext :** +``` +Scheduler → Subscription.process() + → Auto-génère Sales Invoice (draft ou submitted) + → Taxes auto-appliquées via template + → Email via Print Format + Email Account + → Payment Entry (PPA/Stripe) auto-réconcilié +``` + +### Nouveau client + +**Legacy :** +``` +account_add.php → INSERT account +delivery_add.php → INSERT delivery +service_add.php → INSERT service + UPDATE radius +invoice_add.php → première facture manuelle +``` + +**ERPNext :** +``` +Customer (create) → Address (link) → Contact (link) +Subscription (create, plans=[Item]) → auto-generates invoices +Webhook → n8n → FreeRADIUS (create user) +``` + +### Ticket support + +**Legacy :** +``` +ticket_new.php → INSERT ticket +ticket_view.php → INSERT ticket_msg (avec base64 images) +Assignation manuelle (assign_to → staff.id) +``` + +**ERPNext :** +``` +Issue (create) → Communication (messages) +Assignment Rule (auto-assignation par type) +SLA tracking natif +File upload pour images (pas de base64) +``` + +--- + +## 6. État de la migration + +| Phase | Statut | Détail | +|---|---|---| +| 1a. Item Groups | ✅ Fait | 34 groupes, 3 parents | +| 1b. Items | ✅ Fait | 833 items, vitesses, quotas, legacy IDs | +| 1c. Custom Fields | ✅ Fait | Item (8), Customer (4), Subscription (3) | +| 2. Customers + Contacts + Addresses | ✅ Fait | 6,667 Customers + ~6,600 Contacts + ~6,700 Addresses via direct PG (migrate_direct.py, ~30s) | +| 3. Tax Templates + Subscription Plans | ✅ Fait | QC TPS 5% + TVQ 9.975%, 92 Subscription Plans | +| 4. Subscriptions | ✅ Fait | 21,876 Subscriptions avec RADIUS data. **Scheduler PAUSED** | +| 5. Opening Balance | ⏳ Planifié | Soldes calculés legacy — prochaine étape | +| 6. Tickets → Issues | ⏳ Phase 2 | 242K tickets, extraction base64 | +| 7. Sync bidirectionnelle | ⏳ Phase 2 | n8n ETL nightly | + +> **⚠ ATTENTION : Le scheduler ERPNext est PAUSED. Ne pas réactiver sans instruction explicite — cela déclencherait la facturation automatique des 21,876 abonnements.** diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md new file mode 100644 index 0000000..19458ee --- /dev/null +++ b/docs/CHANGELOG.md @@ -0,0 +1,111 @@ +# Changelog — Migration Legacy → ERPNext + +## 2026-03-28 — Phase 1 : Données maîtres + +### Infrastructure +- **Frappe Assistant Core v2.3.3** installé sur ERPNext — MCP connecté à Claude Code +- **Connexion MariaDB legacy** (10.100.80.100) → ERPNext (10.100.5.61) établie + - GRANT SELECT pour `facturation@10.100.5.61` sur `gestionclient` +- **2 Workspaces** créés sur ERPNext Desk : Dispatch (icône tool) + Gigafibre FSM (icône map) + +### Item Groups (34) +Hiérarchie créée sous 3 parents : +- **Services** (16) : Mensualités fibre/sans fil/télévision, Téléphonie, IP Fixe, Hébergement, Nom de domaine, Location P2P, Internet camping, Cloud, Site internet, Services info, Location espace, Garantie, Téléchargement supp, Honoraires +- **Products** (8) : Installation fibre/sans fil/télé, Équipement fibre/sans fil, Quincaillerie, Pièces info, Technicien +- **Frais et ajustements** (10) : Intérêts, Activation, Frais divers taxables, Créances, Recouvrement, Transport, Impression, Infographie, SPECIAL + +### Items (833) +- **833/833** produits importés via script Python (`/tmp/import_items.py`) +- Source : `gestionclient.product` → ERPNext `Item` +- Mapping : SKU → item_code, price → standard_rate, category → item_group +- Items sans services actifs ET inactive dans legacy → `disabled=1` +- Script exécuté depuis le container `erpnext-backend-1` + +### Custom Fields créés + +**Sur Item (section "ISP Settings")** : +| Fieldname | Type | Description | +|-----------|------|-------------| +| legacy_product_id | Int | ID dans gestionclient.product | +| download_speed | Int | Vitesse download en Kbps | +| upload_speed | Int | Vitesse upload en Kbps | +| quota_day_gb | Float | Quota jour en GB | +| quota_night_gb | Float | Quota nuit en GB | +| fibre_lineprofile | Data | Profil OLT ligne | +| fibre_serviceprofile | Data | Profil OLT service | + +**Sur Customer (section "Legacy Settings")** : +| Fieldname | Type | Description | +|-----------|------|-------------| +| legacy_account_id | Int | ID dans gestionclient.account | +| legacy_customer_id | Data | customer_id legacy (ex: DR2, LOUIS4502470070) | +| ppa_enabled | Check | Paiement pré-autorisé AccesD actif | +| stripe_id | Data | Stripe customer ID | + +**Sur Subscription (section "ISP Settings")** : +| Fieldname | Type | Description | +|-----------|------|-------------| +| radius_user | Data | Utilisateur RADIUS (ex: tci44166) | +| radius_pwd | Data | Mot de passe RADIUS | +| legacy_service_id | Int | ID dans gestionclient.service | + +### Données mises à jour +- **833** Items : legacy_product_id peuplé +- **98** Items : vitesses download/upload + quotas en GB + profils fibre + +--- + +## 2026-03-28 — Phase 2 : Customers, Contacts, Addresses + +### Méthode +- **Direct PostgreSQL** (`migrate_direct.py`) au lieu de l'API REST + - Temps d'exécution : ~30 secondes vs plusieurs heures via API REST + - PostgreSQL `max_connections` augmenté à **200** pour supporter la charge + +### Résultats +- **6,667 Customers** importés (comptes actifs + suspendus) +- **~6,600 Contacts** créés avec email, téléphone, cellulaire +- **~6,700 Addresses** créées avec géolocalisation (latitude/longitude) +- Liens Customer ↔ Contact ↔ Address établis + +--- + +## 2026-03-28 — Phase 3 : Tax Templates + Subscription Plans + +### Tax Templates +- **QC TPS 5%** — Sales Taxes and Charges Template +- **QC TVQ 9.975%** — Sales Taxes and Charges Template + +### Subscription Plans +- **92 Subscription Plans** créés à partir des produits actifs avec services +- Mapping : Item → Subscription Plan (billing_interval=Month, currency=CAD) + +--- + +## 2026-03-28 — Phase 4 : Subscriptions + +### Résultats +- **21,876 Subscriptions** importées avec données RADIUS (radius_user, radius_pwd) +- legacy_service_id peuplé pour chaque Subscription +- Lien Customer ↔ Subscription Plan établi + +### ATTENTION CRITIQUE +- **Le scheduler ERPNext est PAUSED** — les Subscriptions ne génèrent PAS de factures automatiquement +- **NE PAS réactiver le scheduler sans instruction explicite** — la réactivation déclencherait la facturation automatique pour les 21,876 abonnements + +--- + +## Connexions et accès + +| Système | Accès | Méthode | +|---------|-------|---------| +| ERPNext API | `token b273a666c86d2d0:06120709db5e414` | REST API | +| ERPNext MCP | Frappe Assistant Core | StreamableHTTP | +| Legacy MariaDB | `facturation@10.100.80.100` | pymysql depuis container ERPNext | +| Legacy SSH | `root@96.125.192.252` (clé SSH copiée) | SSH | +| DB server SSH | `root@10.100.80.100` via sshpass depuis legacy | SSH | + +## Scripts de migration +- `/tmp/import_items.py` — Import 833 produits → Items +- `/tmp/update_items_speeds.py` — MAJ vitesses/quotas sur Items +- Tous exécutés depuis `erpnext-backend-1` container diff --git a/docs/MIGRATION-PLAN.md b/docs/MIGRATION-PLAN.md new file mode 100644 index 0000000..b895924 --- /dev/null +++ b/docs/MIGRATION-PLAN.md @@ -0,0 +1,223 @@ +# Plan de migration — Legacy Facturation → ERPNext + +## 1. Portrait du système legacy + +### Base de données : `gestionclient` (MariaDB sur 10.100.80.100) + +| Table | Records | Description | +|-------|---------|-------------| +| account | 15,303 | Clients (6,510 actifs, 8,594 terminés) | +| delivery | 17,114 | Adresses d'installation (GPS inclus) | +| service | 66,879 | Abonnements (39,624 actifs) | +| product | 833 | Catalogue forfaits/produits | +| product_cat | 34 | Catégories produits | +| invoice | 629,944 | Factures | +| invoice_item | 1,859,260 | Lignes de facture | +| payment | 540,522 | Paiements | +| payment_item | 684,778 | Application paiements → factures | +| compta_journal_ecriture | 1,211,991 | Écritures comptables | +| compta_comptes | 48 | Plan comptable | +| ticket | 242,618 | Tickets support/install/facturation | +| ticket_msg | 784,290 | Messages tickets (8,123 avec images base64) | +| device | 10,377 | Équipements réseau | +| fibre | 16,057 | Raccordements fibre (OLT, VLAN, ports) | +| staff | 155 | Employés | +| bon_travail | 14,472 | Bons de travail technicien | +| tax | 4 | TPS/TVQ | + +### Revenu mensuel actif (~$742K/mois) + +| Catégorie | Services actifs | Revenu mensuel | +|-----------|----------------|----------------| +| Mensualités fibre | 17,050 | $454,364 | +| Mensualités sans fil | 4,200 | $189,799 | +| Mensualités télévision | 2,001 | $46,360 | +| Téléphonie | 2,236 | $34,004 | +| Installation fibre | 10,926 | $7,285 | +| Adresse IP fixe | 138 | $3,485 | +| Autres | ~70 | ~$7K | + +### Relations entre tables + +``` +account (client) + ├── 1:N delivery (adresses d'installation, GPS) + │ └── 1:N fibre (raccordement OLT, VLAN, ports) + ├── 1:N service (abonnements actifs) + │ ├── N:1 product (forfait, prix, vitesses) + │ ├── N:1 delivery (lieu du service) + │ └── N:1 device (équipement assigné) + ├── 1:N invoice (factures) + │ └── 1:N invoice_item (lignes → product) + ├── 1:N payment (paiements) + │ └── 1:N payment_item (répartition sur factures) + └── 1:N ticket (support, install, réparation) + └── 1:N ticket_msg (messages, images base64) + +product + ├── N:1 product_cat (catégorie → num_compte) + └── vitesses (download_speed, upload_speed, quota) + +compta_comptes (plan comptable, 48 comptes) + └── 1:N compta_journal_ecriture (1.2M écritures) + +staff (155 employés) + └── ticket.assign_to, bon_travail.tech1/tech2 +``` + +--- + +## 2. Mapping Legacy → ERPNext + +### Phase A : Données maîtres (référence) + +| Legacy | ERPNext | Notes | +|--------|---------|-------| +| `account` | **Customer** | customer_id, company, group_id, statut, PPA, Stripe | +| `account` (adresse) | **Address** | address1, city, zip, linked to Customer | +| `account` (contact) | **Contact** | first_name, last_name, email, tel, cell | +| `delivery` | **Address** (type=Shipping) + **Service Location** (FSM) | GPS, name, lié au Customer | +| `product` | **Item** (type=Service) | SKU, prix, vitesses dans custom fields | +| `product_cat` | **Item Group** | Hiérarchie catégories, num_compte pour comptabilité | +| `tax` / `tax_group` | **Tax Template** | TPS 5% + TVQ (8.925% → 9.975%) | +| `staff` | **Employee** | username, email, dept | +| `device` | **Asset** ou **Service Equipment** (FSM) | SN, MAC, modèle, parent device | +| `fibre` | Champs dans **Service Location** | OLT info, VLAN, frame/slot/port | +| `compta_comptes` | **Chart of Accounts** | 48 comptes → plan comptable ERPNext | +| `ticket_dept` | **Issue Type** ou **Department** | 21 départements de tickets | + +### Phase B : Données transactionnelles + +| Legacy | ERPNext | Volume | Notes | +|--------|---------|--------|-------| +| `invoice` + `invoice_item` | **Sales Invoice** + Items | 630K factures | Importer seulement les 2-3 dernières années ? | +| `payment` + `payment_item` | **Payment Entry** + References | 540K | Idem — soldes historiques en opening balance | +| `service` (actifs) | **Subscription** | 39,624 | Critique — abonnements récurrents en cours | +| `compta_journal_ecriture` | **Journal Entry** | 1.2M | Opening balance uniquement — pas d'import ligne par ligne | +| `ticket` + `ticket_msg` | **Issue** ou import dans HD | 242K | Les fermés → archive, ouverts → migration | +| `bon_travail` | **Dispatch Job** (FSM) | 14K | Historique tech | + +### Phase C : Fichiers et images + +| Source | Volume | Stratégie | +|--------|--------|-----------| +| `ticket_msg` base64 images | ~8,123 messages | Extraire → fichier → upload ERPNext File → remplacer dans texte | +| `intranet_doc` attachments | ~33 docs | Copier fichiers → ERPNext File | +| `uploads/` directory | À vérifier | Copier vers ERPNext sites/[site]/public/files/ | + +--- + +## 3. Stratégie de migration + +### Principes + +1. **Ne pas tout migrer** — Les données historiques fermées (factures payées, tickets clos) restent consultables dans le legacy en lecture seule +2. **Opening balance** — Un seul Journal Entry avec les soldes de chaque compte au jour J de migration +3. **Services actifs = priorité #1** — Les 39,624 abonnements actifs doivent être fonctionnels dans ERPNext avant le cutover +4. **Coexistence temporaire** — Les deux systèmes roulent en parallèle pendant la phase de validation (1-2 mois) +5. **Images base64** — Extraites en fichiers, uploadées dans ERPNext, lien URL dans le champ texte + +### Phases d'exécution + +#### Phase 1 : Fondations (Semaine 1-2) +- [ ] Créer le Chart of Accounts dans ERPNext (48 comptes) +- [ ] Configurer les Tax Templates (TPS 5% + TVQ 9.975%) +- [ ] Créer les Item Groups (34 catégories produits) +- [ ] Importer les 833 Items (produits/forfaits) avec custom fields (vitesses, quota, profils OLT) +- [ ] Configurer les Pricing Rules si nécessaire + +#### Phase 2 : Clients et adresses (Semaine 2-3) +- [ ] Importer les Customers actifs (6,510) avec mapping group_id → Customer Group +- [ ] Importer les Addresses (delivery) avec GPS → Service Location (FSM) +- [ ] Importer les Contacts (email, téléphones) +- [ ] Lier fibre data aux Service Locations (OLT, VLAN, ports) +- [ ] Importer les Employees (155 staff) + +#### Phase 3 : Abonnements actifs (Semaine 3-4) +- [ ] Créer les Subscription Plans dans ERPNext +- [ ] Importer les 39,624 services actifs comme Subscriptions +- [ ] Valider : chaque service → bon Customer, bon Item, bon prix, bonne adresse +- [ ] Gérer les `hijack` (prix spéciaux par service) → Pricing Rule ou custom field + +#### Phase 4 : Soldes et comptabilité (Semaine 4-5) +- [ ] Calculer les soldes de chaque compte client (factures - paiements) +- [ ] Créer l'Opening Balance dans ERPNext (1 Journal Entry) +- [ ] Valider que le total des comptes à recevoir correspond +- [ ] Importer les factures des 12 derniers mois (optionnel, pour historique) + +#### Phase 5 : Cutover (Semaine 5-6) +- [ ] Freeze le legacy (lecture seule) +- [ ] Run final delta sync (nouvelles transactions depuis dernier import) +- [ ] Activer la facturation récurrente dans ERPNext +- [ ] Rediriger les flux PPA/Stripe vers ERPNext +- [ ] Formation utilisateurs +- [ ] Go-live + +#### Phase 6 : Post-migration (Semaine 7+) +- [ ] Migrer les tickets ouverts (904 open + 487 pending) +- [ ] Extraire et uploader les images base64 des ticket_msg +- [ ] Legacy en mode archive (consultation seulement) +- [ ] Supprimer les accès écriture au legacy + +--- + +## 4. Risques et décisions + +### Décisions à prendre + +1. **Historique des factures** — Combien d'années importer ? Suggestion : 2 ans pour référence, opening balance pour le reste +2. **Tickets historiques** — Garder le legacy en consultation ou tout migrer vers Issue ? +3. **PPA (pré-autorisé bancaire)** — ERPNext supporte GoCardless/Stripe mais pas AccesD directement. Middleware requis ? +4. **Numérotation** — Garder les IDs legacy (customer_id, invoice_id) dans un champ `legacy_id` pour traçabilité ? +5. **Portail client** — account_profile (659 profils) → ERPNext Portal ou www.gigafibre.ca custom ? +6. **Camping** — Tables `camping_*` encore utilisées ? +7. **RADIUS** — Les services ont `radius_user/radius_pwd`. Intégration FreeRADIUS → ERPNext ? + +### Risques + +| Risque | Impact | Mitigation | +|--------|--------|------------| +| Soldes incorrects après migration | Clients facturés en double ou manquant | Réconciliation avant/après avec rapport | +| PPA cassé pendant cutover | Perte de revenus | Coexistence : legacy facture, ERPNext prend le relais progressivement | +| Abonnements manquants | Client sans service | Script de validation : service legacy actif → Subscription ERPNext existe | +| Images base64 perdues | Contexte tickets perdu | Extraction batch avant cutover | +| Performance ERPNext avec 630K factures | Lenteur | Import seulement 2 ans, opening balance pour le reste | + +--- + +## 5. Scripts de migration (à développer) + +``` +migration/ + 01_chart_of_accounts.py # compta_comptes → Chart of Accounts + 02_tax_templates.py # tax → Tax Templates + 03_item_groups.py # product_cat → Item Group + 04_items.py # product → Item + 05_customers.py # account → Customer + Address + Contact + 06_service_locations.py # delivery + fibre → Service Location + 07_employees.py # staff → Employee + 08_equipment.py # device → Service Equipment + 09_subscriptions.py # service (actifs) → Subscription + 10_opening_balance.py # Soldes calculés → Journal Entry + 11_recent_invoices.py # invoice (2 ans) → Sales Invoice + 12_recent_payments.py # payment (2 ans) → Payment Entry + 13_open_tickets.py # ticket (ouverts) → Issue + 14_extract_base64_images.py # ticket_msg base64 → fichiers + 15_validation.py # Réconciliation legacy vs ERPNext +``` + +Chaque script utilise le **MCP ERPNext** (Frappe Assistant Core) pour créer les documents directement, avec validation et rollback en cas d'erreur. + +--- + +## 6. Données à NE PAS migrer + +- `compta_journal_ecriture` complet (1.2M) → opening balance seulement +- `invoice` > 2 ans → opening balance +- `ticket` fermés (241K) → consultables dans legacy +- `ticket_msg` fermés → consultables dans legacy +- `conso_*` (consommation RADIUS) → données opérationnelles, pas comptables +- `phonecall_log_*` → historique téléphonique +- `tmp*`, `*_bk`, `*_archive` → backup tables +- `camping_*` → à confirmer si encore utilisé +- `passwords_manager*` → ne jamais migrer les mots de passe tiers diff --git a/scripts/migration/import_items.py b/scripts/migration/import_items.py new file mode 100644 index 0000000..54ae175 --- /dev/null +++ b/scripts/migration/import_items.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Migration Script: Legacy product -> ERPNext Item +Python 3.5 compatible (no f-strings) +""" +import json +import sys +import requests +import pymysql + +try: + from html import unescape +except ImportError: + try: + from HTMLParser import HTMLParser + unescape = HTMLParser().unescape + except: + unescape = lambda x: x + +LEGACY = { + "host": "10.100.80.100", + "user": "facturation", + "password": "VD67owoj", + "database": "gestionclient", +} + +ERP_URL = "https://erp.gigafibre.ca" +ERP_TOKEN = "token b273a666c86d2d0:06120709db5e414" + +CAT_MAP = { + 1: u"Garantie prolongée", + 2: u"Intérêts et frais divers", + 3: u"Impression", + 4: u"Mensualités sans fil", + 5: u"Technicien", + 6: u"Frais d'activation", + 7: u"Equipement internet sans fil", + 8: u"Installation et équipement internet sans fil", + 9: u"Téléphonie", + 10: u"Site internet", + 11: u"Nom de domaine", + 12: u"Services informatique", + 13: u"Location d'espace", + 14: u"Pièces informatique", + 15: u"Hébergement", + 16: u"Téléchargement supplémentaire", + 17: u"Adresse IP Fixe", + 18: u"Infographie", + 19: u"Revenus - Frais de recouvrement", + 20: u"Créances irrécouvrables", + 21: u"Location point à point", + 22: u"Frais pour irrécouvrables", + 23: u"Internet camping", + 24: u"Transport", + 25: u"Frais divers taxables", + 26: u"Installation et équipement fibre", + 27: u"SPECIAL", + 28: u"Quincaillerie", + 29: u"Equipement internet fibre", + 30: u"Location espace cloud", + 31: u"Honoraires", + 32: u"Mensualités fibre", + 33: u"Mensualités télévision", + 34: u"Installation et équipement télé", +} + +SERVICE_CATS = {1,2,4,6,9,11,12,13,15,16,17,18,19,20,21,22,23,24,25,27,30,31,32,33} + +def erp_post(doctype, data): + url = "{}/api/resource/{}".format(ERP_URL, doctype) + headers = {"Authorization": ERP_TOKEN, "Content-Type": "application/json"} + r = requests.post(url, headers=headers, json={"data": json.dumps(data)}, timeout=30, verify=True) + try: + return r.status_code, r.json() + except: + return r.status_code, {"error": r.text[:200]} + +def main(): + conn = pymysql.connect(**LEGACY) + cur = conn.cursor(pymysql.cursors.DictCursor) + + cur.execute(""" + SELECT p.*, + (SELECT COUNT(*) FROM service s WHERE s.product_id = p.id AND s.status = 1) as active_services + FROM product p + ORDER BY p.id + """) + products = cur.fetchall() + + created = 0 + skipped = 0 + errors = 0 + + for p in products: + sku = p["sku"] or "LEGACY-{}".format(p["id"]) + cat_id = p["category"] or 4 + item_group = CAT_MAP.get(cat_id, "Products") + sku = unescape(sku).strip() + + is_service = cat_id in SERVICE_CATS + disabled = 1 if (not p["active"] and p["active_services"] == 0) else 0 + + item_data = { + "doctype": "Item", + "item_code": sku, + "item_name": sku, + "item_group": item_group, + "stock_uom": "Nos", + "is_stock_item": 0 if is_service else 1, + "disabled": disabled, + "description": "Legacy product ID: {}".format(p["id"]), + "standard_rate": float(p["price"] or 0), + } + + status, resp = erp_post("Item", item_data) + resp_str = str(resp) + + if status == 200 and resp.get("data"): + created += 1 + print(" OK {:<30s} ${:>8.2f} grp={:<25s} svc={}".format( + sku, float(p["price"] or 0), item_group[:25], p["active_services"])) + elif "DuplicateEntryError" in resp_str or "already exists" in resp_str.lower(): + skipped += 1 + print(" SKIP {:<30s} (already exists)".format(sku)) + else: + errors += 1 + err_msg = str(resp.get("exc_type", resp.get("message", resp_str)))[:80] + print(" ERR {:<30s} -> {}".format(sku, err_msg)) + + conn.close() + print("\n=== DONE: {} created, {} skipped, {} errors (total {}) ===".format( + created, skipped, errors, len(products))) + +if __name__ == "__main__": + main() diff --git a/scripts/migration/migrate_direct.py b/scripts/migration/migrate_direct.py new file mode 100644 index 0000000..321167d --- /dev/null +++ b/scripts/migration/migrate_direct.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +Direct PostgreSQL migration: Legacy account → ERPNext Customer + Contact + Address. +Writes directly to tabCustomer, tabContact, tabAddress, tabDynamic Link, etc. +No Frappe ORM, no HTTP API — fastest possible method. + +Run inside erpnext-backend-1: + python3 /tmp/migrate_direct.py + +Idempotent: skips customers where legacy_account_id already exists. +""" +import pymysql +import psycopg2 +import uuid +from html import unescape +from datetime import datetime + +# Legacy MariaDB +LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj", + "database": "gestionclient", "connect_timeout": 30, "read_timeout": 300} + +# ERPNext PostgreSQL +PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123", + "dbname": "_eb65bdc0c4b1b2d6"} + +GROUP_MAP = {1: "Individual", 4: "Commercial", 5: "Individual", 6: "Individual", + 7: "Individual", 8: "Commercial", 9: "Government", 10: "Non Profit"} + +ADMIN = "Administrator" + +def clean(val): + if not val: + return "" + return unescape(str(val)).strip() + +def uid(prefix=""): + return prefix + uuid.uuid4().hex[:10] + +def now(): + return datetime.utcnow() + +def main(): + print("=== Direct PostgreSQL Migration v5 ===") + + # 1. Read all legacy data in one shot + print("Reading legacy MariaDB...", flush=True) + mc = pymysql.connect(**LEGACY) + cur = mc.cursor(pymysql.cursors.DictCursor) + cur.execute("SELECT * FROM account WHERE status IN (1,2) ORDER BY id") + accounts = cur.fetchall() + cur.execute("SELECT * FROM delivery ORDER BY account_id") + all_del = cur.fetchall() + mc.close() + print(" {} accounts, {} deliveries loaded. MariaDB closed.".format(len(accounts), len(all_del)), flush=True) + + # Delivery lookup + active_ids = set(a["id"] for a in accounts) + del_by = {} + for d in all_del: + if d["account_id"] in active_ids: + del_by.setdefault(d["account_id"], []).append(d) + + # 2. Connect ERPNext PostgreSQL + print("Connecting to ERPNext PostgreSQL...", flush=True) + pg = psycopg2.connect(**PG) + pgc = pg.cursor() + + # Existing customers + pgc.execute('SELECT legacy_account_id FROM "tabCustomer" WHERE legacy_account_id IS NOT NULL AND legacy_account_id > 0') + existing = set(r[0] for r in pgc.fetchall()) + print(" {} already imported, will skip".format(len(existing)), flush=True) + + ts = now() + c_ok = c_addr = c_contact = c_skip = c_err = 0 + + for i, a in enumerate(accounts): + aid = a["id"] + if aid in existing: + c_skip += 1 + continue + + first = clean(a["first_name"]) + last = clean(a["last_name"]) + company = clean(a["company"]) + if company: + ctype, cname = "Company", company + else: + ctype, cname = "Individual", "{} {}".format(first, last).strip() or "Client-{}".format(aid) + + cust_id = uid("CUST-") + group = GROUP_MAP.get(a["group_id"], "Individual") + lang = "fr" if a.get("language_id") == "francais" else "en" + disabled = 0 if a["status"] == 1 else 1 + + try: + # --- Customer --- + pgc.execute(""" + INSERT INTO "tabCustomer" ( + name, creation, modified, modified_by, owner, docstatus, idx, + naming_series, customer_name, customer_type, customer_group, + territory, default_currency, language, disabled, + legacy_account_id, legacy_customer_id, ppa_enabled, stripe_id + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + 'CUST-.YYYY.-', %s, %s, %s, + 'Canada', 'CAD', %s, %s, + %s, %s, %s, %s + ) + """, (cust_id, ts, ts, ADMIN, ADMIN, + cname, ctype, group, lang, disabled, + aid, clean(a.get("customer_id")), + 1 if a.get("ppa") else 0, + clean(a.get("stripe_id")) or None)) + c_ok += 1 + + # --- Contact --- + email = clean(a.get("email")) + tel = clean(a.get("tel_home")) + cell = clean(a.get("cell")) + if first or email: + cont_id = uid("CONT-") + full = "{} {}".format(first, last).strip() + pgc.execute(""" + INSERT INTO "tabContact" ( + name, creation, modified, modified_by, owner, docstatus, idx, + first_name, last_name, full_name, email_id, phone, mobile_no, status + ) VALUES (%s, %s, %s, %s, %s, 0, 0, %s, %s, %s, %s, %s, %s, 'Open') + """, (cont_id, ts, ts, ADMIN, ADMIN, + first or cname, last or None, full or cname, + email or None, tel or None, cell or None)) + + # Contact → Customer link (Dynamic Link) + pgc.execute(""" + INSERT INTO "tabDynamic Link" ( + name, creation, modified, modified_by, owner, docstatus, idx, + link_doctype, link_name, link_title, parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, 1, 'Customer', %s, %s, %s, 'links', 'Contact') + """, (uid("DL-"), ts, ts, ADMIN, ADMIN, cust_id, cname, cont_id)) + + # Contact Email (child table) + if email: + pgc.execute(""" + INSERT INTO "tabContact Email" ( + name, creation, modified, modified_by, owner, docstatus, idx, + email_id, is_primary, parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, 1, %s, 1, %s, 'email_ids', 'Contact') + """, (uid("CE-"), ts, ts, ADMIN, ADMIN, email, cont_id)) + + # Contact Phone (child table) + pidx = 1 + if tel: + pgc.execute(""" + INSERT INTO "tabContact Phone" ( + name, creation, modified, modified_by, owner, docstatus, idx, + phone, is_primary_phone, is_primary_mobile_no, + parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 1, 0, %s, 'phone_nos', 'Contact') + """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, tel, cont_id)) + pidx += 1 + if cell: + pgc.execute(""" + INSERT INTO "tabContact Phone" ( + name, creation, modified, modified_by, owner, docstatus, idx, + phone, is_primary_phone, is_primary_mobile_no, + parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, %s, %s, 0, 1, %s, 'phone_nos', 'Contact') + """, (uid("CP-"), ts, ts, ADMIN, ADMIN, pidx, cell, cont_id)) + + c_contact += 1 + + # --- Addresses --- + for j, d in enumerate(del_by.get(aid, [])): + addr1 = clean(d.get("address1")) + city = clean(d.get("city")) + if not addr1 and not city: + continue + + addr_id = uid("ADDR-") + title = clean(d.get("name")) or cname + + pgc.execute(""" + INSERT INTO "tabAddress" ( + name, creation, modified, modified_by, owner, docstatus, idx, + address_title, address_type, address_line1, city, state, + pincode, country, is_primary_address, is_shipping_address + ) VALUES (%s, %s, %s, %s, %s, 0, 0, + %s, 'Shipping', %s, %s, %s, %s, 'Canada', %s, 1) + """, (addr_id, ts, ts, ADMIN, ADMIN, + title, addr1 or "N/A", city or "N/A", + clean(d.get("state")) or "QC", clean(d.get("zip")), + 1 if j == 0 else 0)) + + # Address → Customer link + pgc.execute(""" + INSERT INTO "tabDynamic Link" ( + name, creation, modified, modified_by, owner, docstatus, idx, + link_doctype, link_name, link_title, parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, %s, 'Customer', %s, %s, %s, 'links', 'Address') + """, (uid("DL-"), ts, ts, ADMIN, ADMIN, j+1, cust_id, cname, addr_id)) + + c_addr += 1 + + except Exception as e: + c_err += 1 + pg.rollback() + if c_err <= 20: + print(" ERR #{} {} -> {}".format(aid, cname[:30], str(e)[:100]), flush=True) + continue + + # Commit every 200 + if c_ok % 200 == 0: + pg.commit() + print(" [{}/{}] cust={} addr={} contact={} skip={} err={}".format( + i+1, len(accounts), c_ok, c_addr, c_contact, c_skip, c_err), flush=True) + + pg.commit() + pg.close() + + print("", flush=True) + print("=" * 60, flush=True) + print("Customers: {} created, {} skipped, {} errors".format(c_ok, c_skip, c_err), flush=True) + print("Contacts: {}".format(c_contact), flush=True) + print("Addresses: {}".format(c_addr), flush=True) + print("=" * 60, flush=True) + print("", flush=True) + print("Next: run 'bench --site erp.gigafibre.ca clear-cache'", flush=True) + +if __name__ == "__main__": + main() diff --git a/scripts/migration/migrate_phase3.py b/scripts/migration/migrate_phase3.py new file mode 100644 index 0000000..583555a --- /dev/null +++ b/scripts/migration/migrate_phase3.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +""" +Phase 3b: Create Subscription Plans from active products. +Phase 4: Create Subscriptions from active services. + +Direct PostgreSQL — runs detached inside erpnext-backend-1. +Log: /tmp/migrate_phase3.log +""" +import pymysql +import psycopg2 +import uuid +from datetime import datetime, timezone + +LEGACY = {"host": "10.100.80.100", "user": "facturation", "password": "VD67owoj", + "database": "gestionclient", "connect_timeout": 30, "read_timeout": 300} +PG = {"host": "db", "port": 5432, "user": "postgres", "password": "123", + "dbname": "_eb65bdc0c4b1b2d6"} + +ADMIN = "Administrator" +COMPANY = "TARGO" +TAX_TEMPLATE = "QC TPS 5% + TVQ 9.975% - T" + +# Recurring categories (monthly billing) +RECURRING_CATS = {4, 9, 17, 21, 32, 33} # sans fil, téléphonie, IP fixe, P2P, fibre, TV + +def uid(prefix=""): + return prefix + uuid.uuid4().hex[:10] + +def now(): + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + +def log(msg): + print(msg, flush=True) + +def main(): + ts = now() + log("=== Phase 3b+4: Subscription Plans + Subscriptions ===") + + # 1. Read legacy data + log("Reading legacy data...") + mc = pymysql.connect(**LEGACY) + cur = mc.cursor(pymysql.cursors.DictCursor) + + # Products that have active services + cur.execute(""" + SELECT p.id, p.sku, p.price, p.category, + COUNT(s.id) as active_count + FROM product p + JOIN service s ON s.product_id = p.id AND s.status = 1 + WHERE p.category IN (4,9,17,21,32,33) + GROUP BY p.id + HAVING active_count > 0 + ORDER BY active_count DESC + """) + products = cur.fetchall() + log(" {} recurring products with active services".format(len(products))) + + # Active services with customer mapping + cur.execute(""" + SELECT s.id as service_id, s.product_id, s.delivery_id, s.device_id, + s.date_orig, s.date_next_invoice, s.payment_recurrence, + s.hijack, s.hijack_price, s.hijack_desc, + s.radius_user, s.radius_pwd, s.date_end_contract, + d.account_id + FROM service s + JOIN delivery d ON s.delivery_id = d.id + WHERE s.status = 1 + AND s.product_id IN (SELECT id FROM product WHERE category IN (4,9,17,21,32,33)) + ORDER BY s.id + """) + services = cur.fetchall() + log(" {} active recurring services".format(len(services))) + + # Product SKU lookup + cur.execute("SELECT id, sku FROM product") + sku_map = {r["id"]: r["sku"] for r in cur.fetchall()} + mc.close() + log(" Legacy DB closed.") + + # 2. Connect PostgreSQL + log("Connecting to ERPNext PostgreSQL...") + pg = psycopg2.connect(**PG) + pgc = pg.cursor() + + # Get customer mapping: legacy_account_id → ERPNext customer name + pgc.execute('SELECT legacy_account_id, name FROM "tabCustomer" WHERE legacy_account_id > 0') + cust_map = {r[0]: r[1] for r in pgc.fetchall()} + log(" {} customers mapped".format(len(cust_map))) + + # Check existing subscription plans + pgc.execute('SELECT plan_name FROM "tabSubscription Plan"') + existing_plans = set(r[0] for r in pgc.fetchall()) + + # Check existing subscriptions by legacy_service_id + pgc.execute('SELECT legacy_service_id FROM "tabSubscription" WHERE legacy_service_id IS NOT NULL AND legacy_service_id > 0') + existing_subs = set(r[0] for r in pgc.fetchall()) + log(" {} plans exist, {} subscriptions exist".format(len(existing_plans), len(existing_subs))) + + # ============================ + # Phase 3b: Subscription Plans + # ============================ + log("") + log("--- Phase 3b: Creating Subscription Plans ---") + plans_created = 0 + + for p in products: + sku = sku_map.get(p["id"], "UNKNOWN") + plan_name = "PLAN-{}".format(sku) + + if plan_name in existing_plans: + continue + + price = float(p["price"] or 0) + plan_id = uid("SP-") + + try: + pgc.execute(""" + INSERT INTO "tabSubscription Plan" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan_name, item, currency, price_determination, cost, + billing_interval, billing_interval_count + ) VALUES (%s, %s, %s, %s, %s, 0, 0, + %s, %s, 'CAD', 'Fixed Rate', %s, 'Month', 1) + """, (plan_id, ts, ts, ADMIN, ADMIN, plan_name, sku, price)) + plans_created += 1 + existing_plans.add(plan_name) + except Exception as e: + pg.rollback() + log(" ERR plan {} -> {}".format(sku, str(e)[:80])) + + pg.commit() + log(" {} Subscription Plans created".format(plans_created)) + + # Build plan lookup: product_id → plan_name + plan_by_product = {} + for p in products: + sku = sku_map.get(p["id"], "UNKNOWN") + plan_by_product[p["id"]] = "PLAN-{}".format(sku) + + # ============================ + # Phase 4: Subscriptions + # ============================ + log("") + log("--- Phase 4: Creating Subscriptions ---") + sub_ok = sub_skip = sub_err = 0 + + for i, s in enumerate(services): + sid = s["service_id"] + + if sid in existing_subs: + sub_skip += 1 + continue + + acct_id = s["account_id"] + cust_name = cust_map.get(acct_id) + if not cust_name: + sub_err += 1 + continue + + product_id = s["product_id"] + plan_name = plan_by_product.get(product_id) + if not plan_name: + sub_err += 1 + continue + + # Convert unix timestamps to dates + start_date = None + if s["date_orig"] and s["date_orig"] > 0: + try: + start_date = datetime.fromtimestamp(s["date_orig"], tz=timezone.utc).strftime("%Y-%m-%d") + except (ValueError, OSError): + start_date = "2020-01-01" + if not start_date: + start_date = "2020-01-01" + + sub_id = uid("SUB-") + + # Discount from hijack + discount_amt = 0 + if s["hijack"] and s["hijack_price"]: + discount_amt = abs(float(s["hijack_price"])) + + try: + pgc.execute(""" + INSERT INTO "tabSubscription" ( + name, creation, modified, modified_by, owner, docstatus, idx, + party_type, party, company, status, + start_date, generate_invoice_at, days_until_due, + follow_calendar_months, generate_new_invoices_past_due_date, + submit_invoice, cancel_at_period_end, + sales_tax_template, + additional_discount_amount, + radius_user, radius_pwd, legacy_service_id + ) VALUES ( + %s, %s, %s, %s, %s, 0, 0, + 'Customer', %s, %s, 'Active', + %s, 'Beginning of the current subscription period', 30, + 1, 1, 0, 0, + %s, %s, + %s, %s, %s + ) + """, (sub_id, ts, ts, ADMIN, ADMIN, + cust_name, COMPANY, + start_date, + TAX_TEMPLATE, + discount_amt, + s.get("radius_user") or None, + s.get("radius_pwd") or None, + sid)) + + # Subscription Plan Detail (child table linking plan to subscription) + pgc.execute(""" + INSERT INTO "tabSubscription Plan Detail" ( + name, creation, modified, modified_by, owner, docstatus, idx, + plan, qty, + parent, parentfield, parenttype + ) VALUES (%s, %s, %s, %s, %s, 0, 1, + %s, 1, + %s, 'plans', 'Subscription') + """, (uid("SPD-"), ts, ts, ADMIN, ADMIN, plan_name, sub_id)) + + sub_ok += 1 + + except Exception as e: + sub_err += 1 + pg.rollback() + if sub_err <= 20: + log(" ERR svc#{} -> {}".format(sid, str(e)[:100])) + continue + + if sub_ok % 500 == 0: + pg.commit() + log(" [{}/{}] created={} skip={} err={}".format( + i+1, len(services), sub_ok, sub_skip, sub_err)) + + pg.commit() + pg.close() + + log("") + log("=" * 60) + log("Subscription Plans: {} created".format(plans_created)) + log("Subscriptions: {} created, {} skipped, {} errors".format(sub_ok, sub_skip, sub_err)) + log("=" * 60) + log("") + log("Next: bench --site erp.gigafibre.ca clear-cache") + +if __name__ == "__main__": + main()