From 105b0b2a51edd2124ca1df2c689935daffbc1e94 Mon Sep 17 00:00:00 2001
From: louispaulb <louis.targo@gmail.com>
Date: Mon, 1 Jun 2026 22:40:47 -0400
Subject: [PATCH] =?UTF-8?q?feat(ops):=20per-address=20competitor=20column?=
 =?UTF-8?q?=20via=20Qu=C3=A9bec=20IHV=20open=20data?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replaces the reCAPTCHA-blocked Cogeco scraper with the authoritative Québec
"Accès Internet haute vitesse" open ArcGIS data (providers declared to the
gov by the providers themselves — validated to match Cogeco's own popup).

- hub lib/serviceability.js: address → ADR (Adresse_complete → IdAdresse +
  Etat_hiv, civic+postal match w/ JS street disambiguation) → FRN table
  (IdAdresse → FRN_nom providers + signup URLs). Referer-gated proxy, disk
  cache (90d), polite rate limit. Routes /serviceability/lookup[-batch].
- ops ReportInternetCherPage: "Concurrence (FSI)" column — provider chips
  (Cogeco highlighted), batch-fetched on demand with progress; "Cogeco
  disponible" summary card = churn-risk count; manual Cogeco verify icon kept.

Validated live: 37 Chemin Noël → Cogeco+Targo, 147 Montée Richard → Targo
only, Repentigny → Bell+Cogeco. Endpoints documented in
memory/reference_quebec_ihv.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 apps/ops/src/api/reports.js                   |  15 ++
 apps/ops/src/pages/ReportInternetCherPage.vue |  89 ++++++-
 services/targo-hub/lib/serviceability.js      | 244 ++++++++++++++++++
 services/targo-hub/server.js                  |   2 +
 4 files changed, 342 insertions(+), 8 deletions(-)
 create mode 100644 services/targo-hub/lib/serviceability.js

diff --git a/apps/ops/src/api/reports.js b/apps/ops/src/api/reports.js
index c551e93..82ee17c 100644
--- a/apps/ops/src/api/reports.js
+++ b/apps/ops/src/api/reports.js
@@ -10,6 +10,21 @@ async function hubFetch (path) {
   return res.json()
 }
 
+/**
+ * Per-address provider lookup (Québec IHV open data) for a batch of addresses.
+ * @param {Array<{key,address1,city,zip}>} items  (max 80 per call)
+ * @returns {Promise<{results: Record<string, {matched,etat_label,providers,cogeco}>}>}
+ */
+export async function lookupServiceabilityBatch (items) {
+  const res = await fetch(HUB + '/serviceability/lookup-batch', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ items }),
+  })
+  if (!res.ok) throw new Error('Serviceability API error: ' + res.status)
+  return res.json()
+}
+
 /**
  * Revenue report — GL entries grouped by Income account and month
  * @param {string} start YYYY-MM-DD
diff --git a/apps/ops/src/pages/ReportInternetCherPage.vue b/apps/ops/src/pages/ReportInternetCherPage.vue
index b038ca4..98978be 100644
--- a/apps/ops/src/pages/ReportInternetCherPage.vue
+++ b/apps/ops/src/pages/ReportInternetCherPage.vue
@@ -4,6 +4,11 @@
       <q-btn flat dense icon="arrow_back" to="/rapports" class="q-mr-sm" />
       <div class="text-h6 text-weight-bold">Clients qui paient cher — Internet</div>
       <q-space />
+      <q-btn flat dense icon="travel_explore" class="q-mr-sm"
+             :label="svcBusy ? `Concurrence ${svcProgress.done}/${svcProgress.total}` : 'Concurrence'"
+             :loading="svcBusy" :disable="!rows.length" @click="fetchServiceability(rows)">
+        <q-tooltip max-width="280px">Vérifie, pour chaque adresse, les fournisseurs Internet disponibles (données ouvertes Québec IHV). Cogeco surligné = alternative existante = risque de churn.</q-tooltip>
+      </q-btn>
       <q-btn flat dense icon="download" label="CSV" :href="csvUrl" :disable="!rows.length" />
     </div>
 
@@ -69,6 +74,13 @@
           <div class="text-h6 text-weight-bold text-positive">{{ formatMoney(totalDiscounts) }}</div>
         </div>
       </div>
+      <div class="col-auto" v-if="svcProgress.total">
+        <div class="ops-card text-center" style="min-width:180px">
+          <div class="text-caption text-grey-6">Cogeco disponible</div>
+          <div class="text-h6 text-weight-bold text-deep-orange-8">{{ cogecoCount }}</div>
+          <div class="text-caption text-grey-6">{{ svcBusy ? `vérifié ${svcProgress.done}/${svcProgress.total}` : 'risque de churn' }}</div>
+        </div>
+      </div>
     </div>
 
     <!-- Data table -->
@@ -145,12 +157,29 @@
         </q-td>
       </template>
 
-      <template #body-cell-cogeco="props">
-        <q-td :props="props">
-          <q-btn dense flat round size="sm" icon="travel_explore" color="indigo-7"
-            @click="checkCogeco(props.row)">
-            <q-tooltip max-width="260px">Vérifier Cogeco — copie l'adresse et ouvre le vérificateur de disponibilité dans un nouvel onglet</q-tooltip>
-          </q-btn>
+      <template #body-cell-concurrence="props">
+        <q-td :props="props" style="min-width:210px">
+          <div class="row items-center no-wrap">
+            <div class="col">
+              <div v-if="svcLoadingOf(props.row)" class="text-grey-5"><q-spinner size="14px" /></div>
+              <template v-else-if="svcOf(props.row)">
+                <div v-if="hasProviders(props.row)" class="q-gutter-xs">
+                  <q-chip v-for="p in svcOf(props.row).providers" :key="p.nom" dense size="sm"
+                    :color="isCogeco(p.nom) ? 'deep-orange-3' : 'blue-grey-2'"
+                    :text-color="isCogeco(p.nom) ? 'deep-orange-10' : 'blue-grey-9'"
+                    :icon="isCogeco(p.nom) ? 'priority_high' : undefined"
+                    :label="p.nom">
+                    <q-tooltip v-if="p.url">{{ p.url }}</q-tooltip>
+                  </q-chip>
+                </div>
+                <span v-else class="text-grey-6 text-caption">{{ svcOf(props.row).etat_label }}</span>
+              </template>
+              <span v-else class="text-grey-5">—</span>
+            </div>
+            <q-btn dense flat round size="xs" icon="open_in_new" color="grey-5" @click="checkCogeco(props.row)">
+              <q-tooltip>Ouvrir Cogeco pour valider manuellement</q-tooltip>
+            </q-btn>
+          </div>
         </q-td>
       </template>
     </q-table>
@@ -165,7 +194,7 @@
 <script setup>
 import { ref, computed } from 'vue'
 import { useQuasar, copyToClipboard } from 'quasar'
-import { fetchOverpricedInternet, overpricedInternetCsvUrl } from 'src/api/reports'
+import { fetchOverpricedInternet, overpricedInternetCsvUrl, lookupServiceabilityBatch } from 'src/api/reports'
 
 const $q = useQuasar()
 const threshold = ref(90)
@@ -177,6 +206,12 @@ const rows = ref([])
 const search = ref('')
 const dataAsOf = ref(null)
 
+// Serviceability (Québec IHV) state — keyed by delivery_id
+const svc = ref({})              // delivery_id -> { matched, etat_label, providers, cogeco }
+const svcLoading = ref({})       // delivery_id -> bool
+const svcBusy = ref(false)
+const svcProgress = ref({ done: 0, total: 0 })
+
 const segmentOptions = [
   { label: 'Résidentiel', value: 'residential' },
   { label: 'Commercial', value: 'commercial' },
@@ -193,12 +228,18 @@ const columns = [
   { name: 'n_lines', label: 'Lignes', field: 'n_lines', align: 'center', sortable: true },
   { name: 'detail', label: 'Détail forfait', field: 'detail', align: 'left' },
   { name: 'contact', label: 'Contact', field: 'email', align: 'left' },
-  { name: 'cogeco', label: 'Concurrent', field: 'delivery_id', align: 'center' },
+  { name: 'concurrence', label: 'Concurrence (FSI)', field: 'delivery_id', align: 'left' },
 ]
 
 const totalNet = computed(() => rows.value.reduce((s, r) => s + Number(r.net_internet || 0), 0))
 const avgNet = computed(() => rows.value.length ? totalNet.value / rows.value.length : 0)
 const totalDiscounts = computed(() => rows.value.reduce((s, r) => s + Number(r.discounts || 0), 0))
+const cogecoCount = computed(() => Object.values(svc.value).filter(v => v && v.cogeco).length)
+
+function isCogeco (nom) { return /cogeco/i.test(nom || '') }
+function svcOf (row) { return svc.value[row.delivery_id] }
+function svcLoadingOf (row) { return !!svcLoading.value[row.delivery_id] }
+function hasProviders (row) { const v = svc.value[row.delivery_id]; return !!(v && v.providers && v.providers.length) }
 
 const csvUrl = computed(() => overpricedInternetCsvUrl({
   threshold: threshold.value, segment: segment.value, addons: addons.value,
@@ -252,8 +293,40 @@ async function checkCogeco (row) {
   })
 }
 
+// Batch-fetch provider availability for the given rows via the hub
+// (Québec IHV open data). Chunked + progressive so the column fills as it goes.
+async function fetchServiceability (rowsList) {
+  const items = (rowsList || []).filter(r => r.address1).map(r => ({
+    key: String(r.delivery_id), address1: r.address1, city: r.city, zip: r.zip,
+  }))
+  if (!items.length) return
+  svcBusy.value = true
+  svc.value = {}
+  const ld = {}; items.forEach(it => { ld[it.key] = true }); svcLoading.value = ld
+  svcProgress.value = { done: 0, total: items.length }
+  const CHUNK = 40
+  try {
+    for (let i = 0; i < items.length; i += CHUNK) {
+      const chunk = items.slice(i, i + CHUNK)
+      try {
+        const { results } = await lookupServiceabilityBatch(chunk)
+        const sv = { ...svc.value }; const ldd = { ...svcLoading.value }
+        for (const [k, v] of Object.entries(results)) { sv[k] = v; ldd[k] = false }
+        svc.value = sv; svcLoading.value = ldd
+      } catch (e) {
+        const ldd = { ...svcLoading.value }; chunk.forEach(c => { ldd[c.key] = false }); svcLoading.value = ldd
+        $q.notify({ type: 'warning', message: 'Concurrence : lot ignoré (' + e.message + ')', timeout: 3000 })
+      }
+      svcProgress.value = { done: Math.min(i + CHUNK, items.length), total: items.length }
+    }
+  } finally {
+    svcBusy.value = false
+  }
+}
+
 async function loadReport () {
   loading.value = true
+  svc.value = {}; svcLoading.value = {}; svcProgress.value = { done: 0, total: 0 }
   try {
     const data = await fetchOverpricedInternet({
       threshold: threshold.value, segment: segment.value, addons: addons.value, limit: 5000,
diff --git a/services/targo-hub/lib/serviceability.js b/services/targo-hub/lib/serviceability.js
new file mode 100644
index 0000000..15a9299
--- /dev/null
+++ b/services/targo-hub/lib/serviceability.js
@@ -0,0 +1,244 @@
+'use strict'
+/**
+ * serviceability.js — per-address Internet provider lookup using Québec's
+ * "Accès Internet haute vitesse" open ArcGIS data (the same data behind the
+ * gouv.qc.ca interactive map). Authoritative: providers are DECLARED to the
+ * government by the providers themselves. Replaces the abandoned
+ * cogeco-checker (Cogeco's own site is gated by reCAPTCHA Enterprise).
+ *
+ * Two public ArcGIS services (full recipe in memory/reference_quebec_ihv.md):
+ *   ADR  (Adresse_S)      → address points: Adresse_complete, Etat_hiv, IdAdresse
+ *   FRN  (Fournisseurs_S) → TABLE keyed by IdAdresse: FRN_nom (provider),
+ *                           FRN_URL_inscrip (signup link), Date_fin
+ * The usrsvcs proxy is REFERER-GATED: send `Referer: https://www.quebec.ca/`
+ * (a direct call 403s). No token, no anti-bot.
+ *
+ * Flow:  address → ADR (civic + postal, JS street disambiguation) → IdAdresse
+ *               → FRN (where IdAdresse=…) → [{nom, url, date_fin}]
+ *
+ * Results are cached on disk (data/serviceability-cache.json) keyed by a
+ * normalized address, so re-loads are instant and we stay a polite consumer
+ * of the gov API (rate-limited, low concurrency).
+ *
+ * Routes:
+ *   POST /serviceability/lookup        { address1, city, zip }      → one result
+ *   POST /serviceability/lookup-batch  { items:[{key,address1,city,zip}] }
+ *   GET  /serviceability/cache-stats
+ */
+
+const fs = require('fs')
+const path = require('path')
+const { json, parseBody, httpRequest, log } = require('./helpers')
+
+const ARCGIS_BASE = 'https://utility.arcgis.com'
+const ADR_PATH = '/usrsvcs/servers/396469b496554883b36948d66eba40f5/rest/services/ADR/FeatureServer/0/query'
+const FRN_PATH = '/usrsvcs/servers/5aa672072a9f43129b97b53d06eb3ae9/rest/services/FRN/FeatureServer/0/query'
+const REFERER = 'https://www.quebec.ca/'
+
+const ETAT_LABEL = { 1: 'Desservie', 2: 'Projet en cours', 3: 'Projet à venir', 4: 'Non admissible' }
+
+const CACHE_FILE = path.join(__dirname, '..', 'data', 'serviceability-cache.json')
+const CACHE_TTL_MS = 90 * 24 * 3600 * 1000 // gov data refreshes periodically; 90d is safe
+const BATCH_MAX = 80 // max items processed per /lookup-batch call (bounds latency)
+
+// ── disk-persisted cache ────────────────────────────────────────────────────
+let cache = new Map()
+try {
+  if (fs.existsSync(CACHE_FILE)) {
+    const obj = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'))
+    cache = new Map(Object.entries(obj))
+    log(`[serviceability] cache loaded: ${cache.size} entries`)
+  }
+} catch (e) { log('[serviceability] cache load failed:', e.message) }
+
+let saveTimer = null
+function saveCacheSoon () {
+  if (saveTimer) return
+  saveTimer = setTimeout(() => {
+    saveTimer = null
+    try {
+      fs.mkdirSync(path.dirname(CACHE_FILE), { recursive: true })
+      fs.writeFileSync(CACHE_FILE, JSON.stringify(Object.fromEntries(cache)))
+    } catch (e) { log('[serviceability] cache save failed:', e.message) }
+  }, 2000)
+}
+
+// ── polite rate limiter (concurrency + min interval) ────────────────────────
+const MAX_CONCURRENT = parseInt(process.env.IHV_MAX_CONCURRENT || '3')
+const MIN_INTERVAL_MS = parseInt(process.env.IHV_MIN_INTERVAL_MS || '120')
+let active = 0
+let lastStart = 0
+const queue = []
+function gate (fn) {
+  return new Promise((resolve, reject) => {
+    const run = () => {
+      active++
+      const wait = Math.max(0, MIN_INTERVAL_MS - (Date.now() - lastStart))
+      setTimeout(() => {
+        lastStart = Date.now()
+        fn().then(resolve, reject).finally(() => {
+          active--
+          if (queue.length) queue.shift()()
+        })
+      }, wait)
+    }
+    if (active >= MAX_CONCURRENT) queue.push(run); else run()
+  })
+}
+
+// ── helpers ─────────────────────────────────────────────────────────────────
+function norm (s) {
+  return String(s || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '')
+    .toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim()
+}
+function sqlEsc (s) { return String(s).replace(/'/g, "''") }
+function parseCivic (address1) {
+  const m = String(address1 || '').trim().match(/^(\d+[a-zA-Z]?)/)
+  return m ? m[1] : null
+}
+// distinctive street words (drop civic + generic street-type words)
+const STREET_TYPES = new Set(['rue', 'ch', 'chemin', 'rang', 'montee', 'mtee', 'boul', 'boulevard',
+  'av', 'ave', 'avenue', 'route', 'rte', 'place', 'pl', 'cote', 'côte', 'terrasse', 'impasse',
+  'croissant', 'allee', 'allée', 'st', 'ste', 'saint', 'sainte', 'de', 'du', 'des', 'la', 'le', 'les'])
+function streetWords (address1) {
+  return norm(address1).split(' ').filter(w => w && !/^\d/.test(w) && !STREET_TYPES.has(w))
+}
+
+async function arcgisQuery (pathBase, params) {
+  const qs = Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join('&')
+  const r = await gate(() => httpRequest(ARCGIS_BASE, `${pathBase}?${qs}`, {
+    method: 'GET', headers: { Referer: REFERER }, timeout: 15000,
+  }))
+  if (r.status !== 200 || !r.data || !Array.isArray(r.data.features)) {
+    const msg = r.data && r.data.error ? r.data.error.message : `HTTP ${r.status}`
+    throw new Error('ArcGIS query failed: ' + msg)
+  }
+  return r.data.features.map(f => f.attributes)
+}
+
+// Resolve an address to a single ADR record (IdAdresse + Etat_hiv).
+async function resolveAddress ({ address1, city, zip }) {
+  const civic = parseCivic(address1)
+  if (!civic) return null
+  const postal = String(zip || '').replace(/\s+/g, '').toUpperCase()
+  const target = norm(`${address1} ${city || ''}`)
+  const myWords = new Set(streetWords(address1))
+
+  // Pick the best candidate by street-token overlap against the input.
+  const best = (rows) => {
+    let top = null, topScore = -1
+    for (const a of rows) {
+      const cand = norm(a.Adresse_complete)
+      const candWords = new Set(streetWords(a.Adresse_complete))
+      let overlap = 0
+      for (const w of myWords) if (candWords.has(w)) overlap++
+      // tie-break: also reward overall string containment of city
+      const cityBonus = city && cand.includes(norm(city)) ? 0.5 : 0
+      const score = overlap + cityBonus + (cand === target ? 5 : 0)
+      if (score > topScore) { topScore = score; top = a }
+    }
+    return top
+  }
+
+  // 1) civic + postal — most reliable (postal is embedded in Adresse_complete).
+  if (postal && /^[A-Z]\d[A-Z]\d[A-Z]\d$/.test(postal)) {
+    const rows = await arcgisQuery(ADR_PATH, {
+      where: `Adresse_complete LIKE '${sqlEsc(civic)} %' AND Adresse_complete LIKE '%${sqlEsc(postal)}'`,
+      outFields: 'IdAdresse,Adresse_complete,Etat_hiv,Type_adresse', returnGeometry: 'false',
+      resultRecordCount: '25', f: 'json',
+    })
+    if (rows.length === 1) return rows[0]
+    if (rows.length > 1) return best(rows)
+  }
+
+  // 2) fallback: civic + distinctive street word, then JS-filter by city.
+  const sw = streetWords(address1).sort((a, b) => b.length - a.length)[0]
+  if (sw && /^[a-z]+$/.test(sw)) {
+    const rows = await arcgisQuery(ADR_PATH, {
+      where: `Adresse_complete LIKE '${sqlEsc(civic)} %' AND UPPER(Adresse_complete) LIKE '%${sqlEsc(sw.toUpperCase())}%'`,
+      outFields: 'IdAdresse,Adresse_complete,Etat_hiv,Type_adresse', returnGeometry: 'false',
+      resultRecordCount: '50', f: 'json',
+    })
+    const inCity = city ? rows.filter(a => norm(a.Adresse_complete).includes(norm(city))) : rows
+    const pool = inCity.length ? inCity : rows
+    if (pool.length) return best(pool)
+  }
+  return null
+}
+
+async function fetchProviders (idAdresse) {
+  const rows = await arcgisQuery(FRN_PATH, {
+    where: `IdAdresse='${sqlEsc(idAdresse)}'`,
+    outFields: 'FRN_nom,FRN_URL_inscrip,Date_fin', returnGeometry: 'false',
+    resultRecordCount: '50', f: 'json',
+  })
+  const seen = new Set(); const out = []
+  for (const a of rows) {
+    const nom = (a.FRN_nom || '').trim()
+    if (!nom || seen.has(nom)) continue
+    seen.add(nom)
+    out.push({ nom, url: a.FRN_URL_inscrip || null, date_fin: a.Date_fin || null })
+  }
+  out.sort((a, b) => a.nom.localeCompare(b.nom, 'fr'))
+  return out
+}
+
+// Main entry: address → { matched, idAdresse, adresse_complete, etat_hiv,
+//   etat_label, providers:[{nom,url,date_fin}], cogeco:bool }
+async function lookupProviders ({ address1, city, zip }) {
+  const key = `${norm(address1)}|${String(zip || '').replace(/\s+/g, '').toUpperCase() || norm(city)}`
+  const hit = cache.get(key)
+  if (hit && (Date.now() - hit.ts) < CACHE_TTL_MS) return { ...hit.v, cached: true }
+
+  let result
+  try {
+    const adr = await resolveAddress({ address1, city, zip })
+    if (!adr) {
+      result = { matched: false, etat_label: 'Adresse introuvable', providers: [], cogeco: false }
+    } else {
+      const providers = await fetchProviders(adr.IdAdresse)
+      result = {
+        matched: true,
+        idAdresse: adr.IdAdresse,
+        adresse_complete: adr.Adresse_complete,
+        etat_hiv: adr.Etat_hiv,
+        etat_label: ETAT_LABEL[adr.Etat_hiv] || String(adr.Etat_hiv),
+        type_adresse: adr.Type_adresse || null,
+        providers,
+        cogeco: providers.some(p => /cogeco/i.test(p.nom)),
+      }
+    }
+  } catch (e) {
+    // Don't cache transient errors — let the next call retry.
+    return { matched: false, error: e.message, etat_label: 'Erreur', providers: [], cogeco: false }
+  }
+
+  cache.set(key, { ts: Date.now(), v: result })
+  saveCacheSoon()
+  return result
+}
+
+// ── HTTP handler ─────────────────────────────────────────────────────────────
+async function handle (req, res, method, path) {
+  if (path === '/serviceability/cache-stats' && method === 'GET') {
+    return json(res, 200, { entries: cache.size, ttl_days: CACHE_TTL_MS / 86400000 })
+  }
+  if (path === '/serviceability/lookup' && method === 'POST') {
+    const b = await parseBody(req)
+    if (!b.address1) return json(res, 400, { error: 'address1 required' })
+    return json(res, 200, await lookupProviders(b))
+  }
+  if (path === '/serviceability/lookup-batch' && method === 'POST') {
+    const b = await parseBody(req)
+    const items = Array.isArray(b.items) ? b.items.slice(0, BATCH_MAX) : []
+    if (!items.length) return json(res, 400, { error: 'items[] required' })
+    const results = await Promise.all(items.map(async (it, i) => {
+      const r = await lookupProviders(it)
+      return [it.key != null ? it.key : i, r]
+    }))
+    return json(res, 200, { results: Object.fromEntries(results), processed: items.length, batch_max: BATCH_MAX })
+  }
+  return json(res, 404, { error: 'Not found' })
+}
+
+module.exports = { handle, lookupProviders }
diff --git a/services/targo-hub/server.js b/services/targo-hub/server.js
index 1efd19a..d239f9f 100644
--- a/services/targo-hub/server.js
+++ b/services/targo-hub/server.js
@@ -122,6 +122,8 @@ const server = http.createServer(async (req, res) => {
     // /reports handler so the /reports/legacy/* prefix isn't swallowed.
     if (path.startsWith('/reports/legacy')) return require('./lib/legacy-reports').handle(req, res, method, path, url)
     if (path.startsWith('/reports')) return require('./lib/reports').handle(req, res, method, path, url)
+    // Per-address competitor/provider lookup via Québec IHV open data (ADR+FRN).
+    if (path.startsWith('/serviceability')) return require('./lib/serviceability').handle(req, res, method, path)
     if (path.startsWith('/campaigns')) return require('./lib/campaigns').handle(req, res, method, path)
     // Gift redirect wrapper — short public URLs in campaign emails that
     // 302 to the underlying Giftbit shortlink (subject to our expiry/revoke).