feat(ops): per-address competitor column via Québec IHV open data

Replaces the reCAPTCHA-blocked Cogeco scraper with the authoritative Québec
"Accès Internet haute vitesse" open ArcGIS data (providers declared to the
gov by the providers themselves — validated to match Cogeco's own popup).

- hub lib/serviceability.js: address → ADR (Adresse_complete → IdAdresse +
  Etat_hiv, civic+postal match w/ JS street disambiguation) → FRN table
  (IdAdresse → FRN_nom providers + signup URLs). Referer-gated proxy, disk
  cache (90d), polite rate limit. Routes /serviceability/lookup[-batch].
- ops ReportInternetCherPage: "Concurrence (FSI)" column — provider chips
  (Cogeco highlighted), batch-fetched on demand with progress; "Cogeco
  disponible" summary card = churn-risk count; manual Cogeco verify icon kept.

Validated live: 37 Chemin Noël → Cogeco+Targo, 147 Montée Richard → Targo
only, Repentigny → Bell+Cogeco. Endpoints documented in
memory/reference_quebec_ihv.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
louispaulb 2026-06-01 22:40:47 -04:00
parent 76573f58e9
commit 105b0b2a51
4 changed files with 342 additions and 8 deletions

View File

@ -10,6 +10,21 @@ async function hubFetch (path) {
return res.json()
}
/**
* Per-address provider lookup (Québec IHV open data) for a batch of addresses.
* @param {Array<{key,address1,city,zip}>} items (max 80 per call)
* @returns {Promise<{results: Record<string, {matched,etat_label,providers,cogeco}>}>}
*/
export async function lookupServiceabilityBatch (items) {
const res = await fetch(HUB + '/serviceability/lookup-batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ items }),
})
if (!res.ok) throw new Error('Serviceability API error: ' + res.status)
return res.json()
}
/**
* Revenue report GL entries grouped by Income account and month
* @param {string} start YYYY-MM-DD

View File

@ -4,6 +4,11 @@
<q-btn flat dense icon="arrow_back" to="/rapports" class="q-mr-sm" />
<div class="text-h6 text-weight-bold">Clients qui paient cher Internet</div>
<q-space />
<q-btn flat dense icon="travel_explore" class="q-mr-sm"
:label="svcBusy ? `Concurrence ${svcProgress.done}/${svcProgress.total}` : 'Concurrence'"
:loading="svcBusy" :disable="!rows.length" @click="fetchServiceability(rows)">
<q-tooltip max-width="280px">Vérifie, pour chaque adresse, les fournisseurs Internet disponibles (données ouvertes Québec IHV). Cogeco surligné = alternative existante = risque de churn.</q-tooltip>
</q-btn>
<q-btn flat dense icon="download" label="CSV" :href="csvUrl" :disable="!rows.length" />
</div>
@ -69,6 +74,13 @@
<div class="text-h6 text-weight-bold text-positive">{{ formatMoney(totalDiscounts) }}</div>
</div>
</div>
<div class="col-auto" v-if="svcProgress.total">
<div class="ops-card text-center" style="min-width:180px">
<div class="text-caption text-grey-6">Cogeco disponible</div>
<div class="text-h6 text-weight-bold text-deep-orange-8">{{ cogecoCount }}</div>
<div class="text-caption text-grey-6">{{ svcBusy ? `vérifié ${svcProgress.done}/${svcProgress.total}` : 'risque de churn' }}</div>
</div>
</div>
</div>
<!-- Data table -->
@ -145,12 +157,29 @@
</q-td>
</template>
<template #body-cell-cogeco="props">
<q-td :props="props">
<q-btn dense flat round size="sm" icon="travel_explore" color="indigo-7"
@click="checkCogeco(props.row)">
<q-tooltip max-width="260px">Vérifier Cogeco copie l'adresse et ouvre le vérificateur de disponibilité dans un nouvel onglet</q-tooltip>
</q-btn>
<template #body-cell-concurrence="props">
<q-td :props="props" style="min-width:210px">
<div class="row items-center no-wrap">
<div class="col">
<div v-if="svcLoadingOf(props.row)" class="text-grey-5"><q-spinner size="14px" /></div>
<template v-else-if="svcOf(props.row)">
<div v-if="hasProviders(props.row)" class="q-gutter-xs">
<q-chip v-for="p in svcOf(props.row).providers" :key="p.nom" dense size="sm"
:color="isCogeco(p.nom) ? 'deep-orange-3' : 'blue-grey-2'"
:text-color="isCogeco(p.nom) ? 'deep-orange-10' : 'blue-grey-9'"
:icon="isCogeco(p.nom) ? 'priority_high' : undefined"
:label="p.nom">
<q-tooltip v-if="p.url">{{ p.url }}</q-tooltip>
</q-chip>
</div>
<span v-else class="text-grey-6 text-caption">{{ svcOf(props.row).etat_label }}</span>
</template>
<span v-else class="text-grey-5"></span>
</div>
<q-btn dense flat round size="xs" icon="open_in_new" color="grey-5" @click="checkCogeco(props.row)">
<q-tooltip>Ouvrir Cogeco pour valider manuellement</q-tooltip>
</q-btn>
</div>
</q-td>
</template>
</q-table>
@ -165,7 +194,7 @@
<script setup>
import { ref, computed } from 'vue'
import { useQuasar, copyToClipboard } from 'quasar'
import { fetchOverpricedInternet, overpricedInternetCsvUrl } from 'src/api/reports'
import { fetchOverpricedInternet, overpricedInternetCsvUrl, lookupServiceabilityBatch } from 'src/api/reports'
const $q = useQuasar()
const threshold = ref(90)
@ -177,6 +206,12 @@ const rows = ref([])
const search = ref('')
const dataAsOf = ref(null)
// Serviceability (Québec IHV) state keyed by delivery_id
const svc = ref({}) // delivery_id -> { matched, etat_label, providers, cogeco }
const svcLoading = ref({}) // delivery_id -> bool
const svcBusy = ref(false)
const svcProgress = ref({ done: 0, total: 0 })
const segmentOptions = [
{ label: 'Résidentiel', value: 'residential' },
{ label: 'Commercial', value: 'commercial' },
@ -193,12 +228,18 @@ const columns = [
{ name: 'n_lines', label: 'Lignes', field: 'n_lines', align: 'center', sortable: true },
{ name: 'detail', label: 'Détail forfait', field: 'detail', align: 'left' },
{ name: 'contact', label: 'Contact', field: 'email', align: 'left' },
{ name: 'cogeco', label: 'Concurrent', field: 'delivery_id', align: 'center' },
{ name: 'concurrence', label: 'Concurrence (FSI)', field: 'delivery_id', align: 'left' },
]
const totalNet = computed(() => rows.value.reduce((s, r) => s + Number(r.net_internet || 0), 0))
const avgNet = computed(() => rows.value.length ? totalNet.value / rows.value.length : 0)
const totalDiscounts = computed(() => rows.value.reduce((s, r) => s + Number(r.discounts || 0), 0))
const cogecoCount = computed(() => Object.values(svc.value).filter(v => v && v.cogeco).length)
function isCogeco (nom) { return /cogeco/i.test(nom || '') }
function svcOf (row) { return svc.value[row.delivery_id] }
function svcLoadingOf (row) { return !!svcLoading.value[row.delivery_id] }
function hasProviders (row) { const v = svc.value[row.delivery_id]; return !!(v && v.providers && v.providers.length) }
const csvUrl = computed(() => overpricedInternetCsvUrl({
threshold: threshold.value, segment: segment.value, addons: addons.value,
@ -252,8 +293,40 @@ async function checkCogeco (row) {
})
}
// Batch-fetch provider availability for the given rows via the hub
// (Québec IHV open data). Chunked + progressive so the column fills as it goes.
async function fetchServiceability (rowsList) {
const items = (rowsList || []).filter(r => r.address1).map(r => ({
key: String(r.delivery_id), address1: r.address1, city: r.city, zip: r.zip,
}))
if (!items.length) return
svcBusy.value = true
svc.value = {}
const ld = {}; items.forEach(it => { ld[it.key] = true }); svcLoading.value = ld
svcProgress.value = { done: 0, total: items.length }
const CHUNK = 40
try {
for (let i = 0; i < items.length; i += CHUNK) {
const chunk = items.slice(i, i + CHUNK)
try {
const { results } = await lookupServiceabilityBatch(chunk)
const sv = { ...svc.value }; const ldd = { ...svcLoading.value }
for (const [k, v] of Object.entries(results)) { sv[k] = v; ldd[k] = false }
svc.value = sv; svcLoading.value = ldd
} catch (e) {
const ldd = { ...svcLoading.value }; chunk.forEach(c => { ldd[c.key] = false }); svcLoading.value = ldd
$q.notify({ type: 'warning', message: 'Concurrence : lot ignoré (' + e.message + ')', timeout: 3000 })
}
svcProgress.value = { done: Math.min(i + CHUNK, items.length), total: items.length }
}
} finally {
svcBusy.value = false
}
}
async function loadReport () {
loading.value = true
svc.value = {}; svcLoading.value = {}; svcProgress.value = { done: 0, total: 0 }
try {
const data = await fetchOverpricedInternet({
threshold: threshold.value, segment: segment.value, addons: addons.value, limit: 5000,

View File

@ -0,0 +1,244 @@
'use strict'
/**
* serviceability.js per-address Internet provider lookup using Québec's
* "Accès Internet haute vitesse" open ArcGIS data (the same data behind the
* gouv.qc.ca interactive map). Authoritative: providers are DECLARED to the
* government by the providers themselves. Replaces the abandoned
* cogeco-checker (Cogeco's own site is gated by reCAPTCHA Enterprise).
*
* Two public ArcGIS services (full recipe in memory/reference_quebec_ihv.md):
* ADR (Adresse_S) address points: Adresse_complete, Etat_hiv, IdAdresse
* FRN (Fournisseurs_S) TABLE keyed by IdAdresse: FRN_nom (provider),
* FRN_URL_inscrip (signup link), Date_fin
* The usrsvcs proxy is REFERER-GATED: send `Referer: https://www.quebec.ca/`
* (a direct call 403s). No token, no anti-bot.
*
* Flow: address ADR (civic + postal, JS street disambiguation) IdAdresse
* FRN (where IdAdresse=) [{nom, url, date_fin}]
*
* Results are cached on disk (data/serviceability-cache.json) keyed by a
* normalized address, so re-loads are instant and we stay a polite consumer
* of the gov API (rate-limited, low concurrency).
*
* Routes:
* POST /serviceability/lookup { address1, city, zip } one result
* POST /serviceability/lookup-batch { items:[{key,address1,city,zip}] }
* GET /serviceability/cache-stats
*/
const fs = require('fs')
const path = require('path')
const { json, parseBody, httpRequest, log } = require('./helpers')
const ARCGIS_BASE = 'https://utility.arcgis.com'
const ADR_PATH = '/usrsvcs/servers/396469b496554883b36948d66eba40f5/rest/services/ADR/FeatureServer/0/query'
const FRN_PATH = '/usrsvcs/servers/5aa672072a9f43129b97b53d06eb3ae9/rest/services/FRN/FeatureServer/0/query'
const REFERER = 'https://www.quebec.ca/'
const ETAT_LABEL = { 1: 'Desservie', 2: 'Projet en cours', 3: 'Projet à venir', 4: 'Non admissible' }
const CACHE_FILE = path.join(__dirname, '..', 'data', 'serviceability-cache.json')
const CACHE_TTL_MS = 90 * 24 * 3600 * 1000 // gov data refreshes periodically; 90d is safe
const BATCH_MAX = 80 // max items processed per /lookup-batch call (bounds latency)
// ── disk-persisted cache ────────────────────────────────────────────────────
let cache = new Map()
try {
if (fs.existsSync(CACHE_FILE)) {
const obj = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'))
cache = new Map(Object.entries(obj))
log(`[serviceability] cache loaded: ${cache.size} entries`)
}
} catch (e) { log('[serviceability] cache load failed:', e.message) }
let saveTimer = null
function saveCacheSoon () {
if (saveTimer) return
saveTimer = setTimeout(() => {
saveTimer = null
try {
fs.mkdirSync(path.dirname(CACHE_FILE), { recursive: true })
fs.writeFileSync(CACHE_FILE, JSON.stringify(Object.fromEntries(cache)))
} catch (e) { log('[serviceability] cache save failed:', e.message) }
}, 2000)
}
// ── polite rate limiter (concurrency + min interval) ────────────────────────
const MAX_CONCURRENT = parseInt(process.env.IHV_MAX_CONCURRENT || '3')
const MIN_INTERVAL_MS = parseInt(process.env.IHV_MIN_INTERVAL_MS || '120')
let active = 0
let lastStart = 0
const queue = []
function gate (fn) {
return new Promise((resolve, reject) => {
const run = () => {
active++
const wait = Math.max(0, MIN_INTERVAL_MS - (Date.now() - lastStart))
setTimeout(() => {
lastStart = Date.now()
fn().then(resolve, reject).finally(() => {
active--
if (queue.length) queue.shift()()
})
}, wait)
}
if (active >= MAX_CONCURRENT) queue.push(run); else run()
})
}
// ── helpers ─────────────────────────────────────────────────────────────────
function norm (s) {
return String(s || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '')
.toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim()
}
function sqlEsc (s) { return String(s).replace(/'/g, "''") }
function parseCivic (address1) {
const m = String(address1 || '').trim().match(/^(\d+[a-zA-Z]?)/)
return m ? m[1] : null
}
// distinctive street words (drop civic + generic street-type words)
const STREET_TYPES = new Set(['rue', 'ch', 'chemin', 'rang', 'montee', 'mtee', 'boul', 'boulevard',
'av', 'ave', 'avenue', 'route', 'rte', 'place', 'pl', 'cote', 'côte', 'terrasse', 'impasse',
'croissant', 'allee', 'allée', 'st', 'ste', 'saint', 'sainte', 'de', 'du', 'des', 'la', 'le', 'les'])
function streetWords (address1) {
return norm(address1).split(' ').filter(w => w && !/^\d/.test(w) && !STREET_TYPES.has(w))
}
async function arcgisQuery (pathBase, params) {
const qs = Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join('&')
const r = await gate(() => httpRequest(ARCGIS_BASE, `${pathBase}?${qs}`, {
method: 'GET', headers: { Referer: REFERER }, timeout: 15000,
}))
if (r.status !== 200 || !r.data || !Array.isArray(r.data.features)) {
const msg = r.data && r.data.error ? r.data.error.message : `HTTP ${r.status}`
throw new Error('ArcGIS query failed: ' + msg)
}
return r.data.features.map(f => f.attributes)
}
// Resolve an address to a single ADR record (IdAdresse + Etat_hiv).
async function resolveAddress ({ address1, city, zip }) {
const civic = parseCivic(address1)
if (!civic) return null
const postal = String(zip || '').replace(/\s+/g, '').toUpperCase()
const target = norm(`${address1} ${city || ''}`)
const myWords = new Set(streetWords(address1))
// Pick the best candidate by street-token overlap against the input.
const best = (rows) => {
let top = null, topScore = -1
for (const a of rows) {
const cand = norm(a.Adresse_complete)
const candWords = new Set(streetWords(a.Adresse_complete))
let overlap = 0
for (const w of myWords) if (candWords.has(w)) overlap++
// tie-break: also reward overall string containment of city
const cityBonus = city && cand.includes(norm(city)) ? 0.5 : 0
const score = overlap + cityBonus + (cand === target ? 5 : 0)
if (score > topScore) { topScore = score; top = a }
}
return top
}
// 1) civic + postal — most reliable (postal is embedded in Adresse_complete).
if (postal && /^[A-Z]\d[A-Z]\d[A-Z]\d$/.test(postal)) {
const rows = await arcgisQuery(ADR_PATH, {
where: `Adresse_complete LIKE '${sqlEsc(civic)} %' AND Adresse_complete LIKE '%${sqlEsc(postal)}'`,
outFields: 'IdAdresse,Adresse_complete,Etat_hiv,Type_adresse', returnGeometry: 'false',
resultRecordCount: '25', f: 'json',
})
if (rows.length === 1) return rows[0]
if (rows.length > 1) return best(rows)
}
// 2) fallback: civic + distinctive street word, then JS-filter by city.
const sw = streetWords(address1).sort((a, b) => b.length - a.length)[0]
if (sw && /^[a-z]+$/.test(sw)) {
const rows = await arcgisQuery(ADR_PATH, {
where: `Adresse_complete LIKE '${sqlEsc(civic)} %' AND UPPER(Adresse_complete) LIKE '%${sqlEsc(sw.toUpperCase())}%'`,
outFields: 'IdAdresse,Adresse_complete,Etat_hiv,Type_adresse', returnGeometry: 'false',
resultRecordCount: '50', f: 'json',
})
const inCity = city ? rows.filter(a => norm(a.Adresse_complete).includes(norm(city))) : rows
const pool = inCity.length ? inCity : rows
if (pool.length) return best(pool)
}
return null
}
async function fetchProviders (idAdresse) {
const rows = await arcgisQuery(FRN_PATH, {
where: `IdAdresse='${sqlEsc(idAdresse)}'`,
outFields: 'FRN_nom,FRN_URL_inscrip,Date_fin', returnGeometry: 'false',
resultRecordCount: '50', f: 'json',
})
const seen = new Set(); const out = []
for (const a of rows) {
const nom = (a.FRN_nom || '').trim()
if (!nom || seen.has(nom)) continue
seen.add(nom)
out.push({ nom, url: a.FRN_URL_inscrip || null, date_fin: a.Date_fin || null })
}
out.sort((a, b) => a.nom.localeCompare(b.nom, 'fr'))
return out
}
// Main entry: address → { matched, idAdresse, adresse_complete, etat_hiv,
// etat_label, providers:[{nom,url,date_fin}], cogeco:bool }
async function lookupProviders ({ address1, city, zip }) {
const key = `${norm(address1)}|${String(zip || '').replace(/\s+/g, '').toUpperCase() || norm(city)}`
const hit = cache.get(key)
if (hit && (Date.now() - hit.ts) < CACHE_TTL_MS) return { ...hit.v, cached: true }
let result
try {
const adr = await resolveAddress({ address1, city, zip })
if (!adr) {
result = { matched: false, etat_label: 'Adresse introuvable', providers: [], cogeco: false }
} else {
const providers = await fetchProviders(adr.IdAdresse)
result = {
matched: true,
idAdresse: adr.IdAdresse,
adresse_complete: adr.Adresse_complete,
etat_hiv: adr.Etat_hiv,
etat_label: ETAT_LABEL[adr.Etat_hiv] || String(adr.Etat_hiv),
type_adresse: adr.Type_adresse || null,
providers,
cogeco: providers.some(p => /cogeco/i.test(p.nom)),
}
}
} catch (e) {
// Don't cache transient errors — let the next call retry.
return { matched: false, error: e.message, etat_label: 'Erreur', providers: [], cogeco: false }
}
cache.set(key, { ts: Date.now(), v: result })
saveCacheSoon()
return result
}
// ── HTTP handler ─────────────────────────────────────────────────────────────
async function handle (req, res, method, path) {
if (path === '/serviceability/cache-stats' && method === 'GET') {
return json(res, 200, { entries: cache.size, ttl_days: CACHE_TTL_MS / 86400000 })
}
if (path === '/serviceability/lookup' && method === 'POST') {
const b = await parseBody(req)
if (!b.address1) return json(res, 400, { error: 'address1 required' })
return json(res, 200, await lookupProviders(b))
}
if (path === '/serviceability/lookup-batch' && method === 'POST') {
const b = await parseBody(req)
const items = Array.isArray(b.items) ? b.items.slice(0, BATCH_MAX) : []
if (!items.length) return json(res, 400, { error: 'items[] required' })
const results = await Promise.all(items.map(async (it, i) => {
const r = await lookupProviders(it)
return [it.key != null ? it.key : i, r]
}))
return json(res, 200, { results: Object.fromEntries(results), processed: items.length, batch_max: BATCH_MAX })
}
return json(res, 404, { error: 'Not found' })
}
module.exports = { handle, lookupProviders }

View File

@ -122,6 +122,8 @@ const server = http.createServer(async (req, res) => {
// /reports handler so the /reports/legacy/* prefix isn't swallowed.
if (path.startsWith('/reports/legacy')) return require('./lib/legacy-reports').handle(req, res, method, path, url)
if (path.startsWith('/reports')) return require('./lib/reports').handle(req, res, method, path, url)
// Per-address competitor/provider lookup via Québec IHV open data (ADR+FRN).
if (path.startsWith('/serviceability')) return require('./lib/serviceability').handle(req, res, method, path)
if (path.startsWith('/campaigns')) return require('./lib/campaigns').handle(req, res, method, path)
// Gift redirect wrapper — short public URLs in campaign emails that
// 302 to the underlying Giftbit shortlink (subject to our expiry/revoke).