diff --git a/.env.example b/.env.example index 0ce1d89b..f0279fc7 100644 --- a/.env.example +++ b/.env.example @@ -517,6 +517,20 @@ PROVIDER_HEALTH_CHECK_CRON=*/5 * * * * # Webhook URLs for provider health alerts (comma-separated or individual) PROVIDER_HEALTH_WEBHOOK_URL= +# --------------------------------------------------------------------------- +# Provider Token Watchdog +# Proactively detects expired/revoked provider credentials (MTN/Airtel/Orange) +# and dead or stale accounting OAuth tokens (Xero/QuickBooks) before they +# interrupt service. Critical findings page PagerDuty (PAGERDUTY_INTEGRATION_KEY); +# stale-token warnings go to the webhook(s) below. +# --------------------------------------------------------------------------- +# Cron schedule (default: every 5 minutes) +PROVIDER_TOKEN_WATCHDOG_CRON=*/5 * * * * +# Webhook URL for warning-level provider token alerts (e.g. Slack) +PROVIDER_TOKEN_ALERT_WEBHOOK_URL= +# Re-alert a stale accounting refresh token at most once per N hours (default: 24) +PROVIDER_TOKEN_STALE_REALERT_HOURS=24 + # --------------------------------------------------------------------------- # PII Encryption (AES-256-GCM) # --------------------------------------------------------------------------- diff --git a/docs/runbooks/01-provider-down.md b/docs/runbooks/01-provider-down.md index cc2ee0f7..db83035a 100644 --- a/docs/runbooks/01-provider-down.md +++ b/docs/runbooks/01-provider-down.md @@ -118,3 +118,28 @@ that just converts fast failures into slow ones and floods retries. - File provider-side incident reference; track their RCA. - **Related:** [04 Queue backlog](./04-queue-backlog.md) (parked payouts), [10 Elevated error rate](./10-elevated-error-rate.md). + +--- + +## Credential expiry (prevention) + +The **provider token watchdog** (`provider-token-watchdog` cron job, every 5 +minutes) catches expired/revoked authentication before it becomes an outage: + +- **Mobile money (MTN/Airtel/Orange):** probes the provider auth endpoint with + real credentials. A `401/403` raises a CRITICAL PagerDuty incident + (`provider rejected our credentials`) instead of being counted as "up" by the + uptime watchdog. Rotate the API key/secret in the secrets store and roll pods + (`scripts/rotate-keys.ts` is for DB encryption keys, not provider secrets). +- **Accounting (Xero/QuickBooks):** + - Access token already expired (scheduled refresh failed) → one auto-heal + refresh attempt; if that fails, a CRITICAL PagerDuty incident fires: + the refresh token has expired or been revoked and the user must + re-authorize via the OAuth connect flow. + - Refresh token stale (approaching the provider inactivity window: Xero + 60 days, QuickBooks 100 days) → warning webhook alert + (`PROVIDER_TOKEN_ALERT_WEBHOOK_URL` / `SLACK_ALERTS_WEBHOOK_URL`) so the + integration is reused or reconnected before the token dies. + +PagerDuty events are deduplicated per provider/connection and auto-resolve when +credentials are refreshed or the connection is reconnected. diff --git a/src/jobs/providerTokenWatchdog.ts b/src/jobs/providerTokenWatchdog.ts new file mode 100644 index 00000000..83421443 --- /dev/null +++ b/src/jobs/providerTokenWatchdog.ts @@ -0,0 +1,563 @@ +import { MTNProvider } from "../services/mobilemoney/providers/mtn"; +import { AirtelService } from "../services/mobilemoney/providers/airtel"; +import { OrangeProvider } from "../services/mobilemoney/providers/orange"; +import { + AccountingConnection, + AccountingProvider, + AccountingService, +} from "../services/accounting"; + +// ============================================================================ +// Provider Token Watchdog +// ============================================================================ +// +// Scheduled every 5 minutes. Detects the ways provider authentication can +// silently break BEFORE it interrupts service: +// +// 1. Mobile money credentials (MTN/Airtel/Orange API keys) revoked or expired: +// probes each provider's auth endpoint with the real credentials and treats +// a 401/403 as "credentials invalid". The uptime watchdog treats any HTTP +// <500 (including 401/403) as healthy, so it cannot see this — an expired +// credential would otherwise only surface when real transactions fail. +// +// 2. Accounting OAuth tokens (Xero / QuickBooks): +// - access token already expired (the scheduled refresh failed) → one +// auto-heal refresh attempt; if that fails, raises a CRITICAL incident +// that manual re-authorization is required; +// - refresh token stale (approaching the provider's inactivity window: +// Xero 60 days, QuickBooks 100 days) → sends a warning webhook alert so +// the integration is reused or reconnected before the token dies. +// +// Critical findings page PagerDuty; warnings go to webhook(s). The PagerDuty +// events use dedup keys so repeated runs do not re-page while an incident is +// active, and incidents auto-resolve once the condition clears. +// ============================================================================ + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export type ProviderName = "mtn" | "airtel" | "orange"; + +interface CredentialProbe { + success: boolean; + invalidCredentials?: boolean; + error?: unknown; +} + +interface IncidentRecord { + subject: string; + triggeredAt: string; + dedupeKey: string; +} + +interface StaleTokenAlertPayload { + alertType: "provider_token_stale"; + severity: "warning"; + generatedAt: string; + connections: Array<{ + connectionId: string; + provider: AccountingProvider; + daysSinceRefresh: number; + refreshTokenLimitDays: number; + action: string; + }>; +} + +// ─── Module-level incident state ────────────────────────────────────────────── +// Persists across cron invocations within the same process so incidents are +// not re-triggered on every run, and resolve when the condition clears. + +function dedupPrefix(): string { + return process.env.PAGERDUTY_DEDUP_KEY ?? "proxypay-token-watchdog"; +} + +const credentialIncidents = new Map(); +const reauthIncidents = new Map(); +const staleWarnedAt = new Map(); + +// ─── Accounting refresh-token windows ───────────────────────────────────────── +// OAuth refresh tokens for accounting providers expire after a period of +// inactivity. Each successful refresh resets the clock; once the window lapses +// the only recovery is a manual re-authorization. + +const XERO_REFRESH_TOKEN_LIMIT_DAYS = 60; +const QUICKBOOKS_REFRESH_TOKEN_LIMIT_DAYS = 100; +const REFRESH_TOKEN_WARN_LEAD_DAYS = 15; // warn this many days before the limit + +const DAY_MS = 24 * 60 * 60 * 1000; + +function refreshTokenLimitDays(provider: AccountingProvider): number { + return provider === AccountingProvider.XERO + ? XERO_REFRESH_TOKEN_LIMIT_DAYS + : QUICKBOOKS_REFRESH_TOKEN_LIMIT_DAYS; +} + +function staleWarnDays(provider: AccountingProvider): number { + return refreshTokenLimitDays(provider) - REFRESH_TOKEN_WARN_LEAD_DAYS; +} + +function staleRealertIntervalMs(): number { + const hours = Number(process.env.PROVIDER_TOKEN_STALE_REALERT_HOURS ?? 24); + return (Number.isFinite(hours) && hours > 0 ? hours : 24) * 60 * 60 * 1000; +} + +// ─── PagerDuty helpers ──────────────────────────────────────────────────────── + +const PAGERDUTY_API = "https://events.pagerduty.com/v2/enqueue"; + +function integrationKey(): string { + return process.env.PAGERDUTY_INTEGRATION_KEY ?? ""; +} + +interface PagerDutyPayload { + routing_key: string; + event_action: "trigger" | "resolve"; + dedup_key: string; + payload: { + summary: string; + timestamp: string; + severity: "critical" | "warning" | "info"; + source: string; + custom_details: Record; + }; +} + +async function sendPagerDutyEvent(body: PagerDutyPayload): Promise { + if (!integrationKey()) { + log( + "warn", + "PAGERDUTY_INTEGRATION_KEY not set — skipping PagerDuty event", + { event_action: body.event_action, dedup_key: body.dedup_key }, + ); + return; + } + + const response = await fetch(PAGERDUTY_API, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + throw new Error( + `PagerDuty API responded with HTTP ${response.status}: ${await response.text()}`, + ); + } +} + +async function triggerPagerDutyIncident( + incident: IncidentRecord, + summary: string, + customDetails: Record, +): Promise { + await sendPagerDutyEvent({ + routing_key: integrationKey(), + event_action: "trigger", + dedup_key: incident.dedupeKey, + payload: { + summary, + timestamp: new Date().toISOString(), + severity: "critical", + source: "provider-token-watchdog", + custom_details: { + environment: process.env.NODE_ENV ?? "development", + ...customDetails, + }, + }, + }); +} + +async function resolvePagerDutyIncident( + incident: IncidentRecord, +): Promise { + await sendPagerDutyEvent({ + routing_key: integrationKey(), + event_action: "resolve", + dedup_key: incident.dedupeKey, + payload: { + summary: `[RESOLVED] ${incident.subject}`, + timestamp: new Date().toISOString(), + severity: "info", + source: "provider-token-watchdog", + custom_details: { environment: process.env.NODE_ENV ?? "development" }, + }, + }); +} + +// ─── Webhook helpers (warning-level alerts) ─────────────────────────────────── + +function resolveWarningWebhookUrls(): string[] { + const values = [ + process.env.PROVIDER_TOKEN_ALERT_WEBHOOK_URL, + process.env.SLACK_ALERTS_WEBHOOK_URL, + ].filter((value): value is string => Boolean(value && value.trim())); + + return [...new Set(values)]; +} + +async function postWebhookAlert( + url: string, + payload: StaleTokenAlertPayload, +): Promise { + const response = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error(`Webhook responded with HTTP ${response.status}`); + } +} + +// ─── Structured logger ──────────────────────────────────────────────────────── + +type LogLevel = "info" | "warn" | "error"; + +function log( + level: LogLevel, + message: string, + meta: Record = {}, +): void { + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + level, + service: "provider-token-watchdog", + message, + ...meta, + }); + if (level === "error") { + console.error(line); + } else if (level === "warn") { + console.warn(line); + } else { + console.log(line); + } +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} + +// ─── Mobile money credential probes ─────────────────────────────────────────── + +const CREDENTIAL_CHECKS: Array<{ + name: ProviderName; + check: () => Promise; +}> = [ + { name: "mtn", check: () => new MTNProvider().checkAuth() }, + { name: "airtel", check: () => new AirtelService().checkAuth() }, + { name: "orange", check: () => new OrangeProvider().checkAuth() }, +]; + +/** + * Probes each mobile money provider's auth endpoint with real credentials. + * + * - 401/403 → trigger a CRITICAL "credentials invalid" incident (deduped per + * provider) and keep it active until a probe succeeds again. + * - success → resolve any active credential incident. + * - other failures (network / 5xx) → ignored here; the uptime watchdog owns + * "provider down" classification. + */ +export async function checkMobileMoneyCredentials(): Promise { + for (const { name, check } of CREDENTIAL_CHECKS) { + let probe: CredentialProbe; + try { + probe = await check(); + } catch (error) { + log("error", "Credential probe crashed", { + provider: name, + reason: toErrorMessage(error), + }); + continue; + } + + if (probe.invalidCredentials) { + const active = credentialIncidents.get(name); + if (active) { + log("warn", "Provider credentials still invalid — incident already active", { + provider: name, + activeSince: active.triggeredAt, + }); + continue; + } + + const now = new Date().toISOString(); + const incident: IncidentRecord = { + subject: `Mobile money provider ${name.toUpperCase()} rejected our credentials`, + triggeredAt: now, + dedupeKey: `${dedupPrefix()}-${name}-credentials`, + }; + + try { + await triggerPagerDutyIncident( + incident, + `[CRITICAL] ${name.toUpperCase()} credentials expired or revoked — manual refresh required`, + { + provider: name, + status: "invalid_credentials", + detectedAt: now, + }, + ); + credentialIncidents.set(name, incident); + log("error", "Provider credential incident triggered", { + provider: name, + dedupeKey: incident.dedupeKey, + }); + } catch (error) { + log("error", "Failed to trigger credential incident", { + provider: name, + reason: toErrorMessage(error), + }); + } + } else if (probe.success) { + const incident = credentialIncidents.get(name); + if (!incident) continue; + + try { + await resolvePagerDutyIncident(incident); + credentialIncidents.delete(name); + log("info", "Provider credential incident resolved", { provider: name }); + } catch (error) { + log("error", "Failed to resolve credential incident", { + provider: name, + reason: toErrorMessage(error), + }); + } + } else { + log("info", "Provider auth unreachable — not a credentials issue", { + provider: name, + }); + } + } +} + +// ─── Accounting token checks ────────────────────────────────────────────────── + +/** + * Checks all active accounting connections for dead or dying OAuth tokens. + * + * - `expires_at` in the past (the scheduled refresh already failed) → one + * auto-heal refresh attempt; on failure, raise a CRITICAL incident that + * manual re-authorization is required (deduped per connection). + * - refresh token stale (no refresh within the provider's inactivity window) + * → warning webhook alert, re-alerted at most once per + * PROVIDER_TOKEN_STALE_REALERT_HOURS. + */ +export async function checkAccountingTokens(): Promise { + const accountingService = new AccountingService(); + + let connections: AccountingConnection[]; + try { + connections = await accountingService.getAllActiveConnections(); + } catch (error) { + log("error", "Failed to load accounting connections", { + reason: toErrorMessage(error), + }); + return; + } + + if (connections.length === 0) { + log("info", "No active accounting connections to check"); + return; + } + + const now = Date.now(); + + for (const connection of connections) { + const expiresAt = connection.expiresAt.getTime(); + + // Resolve a previous re-authorization incident once the connection heals + // (e.g. the user reconnected through the OAuth flow). + const activeReauth = reauthIncidents.get(connection.id); + if (activeReauth && expiresAt > now) { + try { + await resolvePagerDutyIncident(activeReauth); + reauthIncidents.delete(connection.id); + log("info", "Accounting re-authorization incident resolved", { + connectionId: connection.id, + }); + } catch (error) { + log("error", "Failed to resolve re-authorization incident", { + connectionId: connection.id, + reason: toErrorMessage(error), + }); + } + } + + if (expiresAt <= now) { + await handleExpiredAccountingToken(accountingService, connection); + continue; + } + + const lastRefreshMs = connection.updatedAt.getTime(); + if (now - lastRefreshMs > staleWarnDays(connection.provider) * DAY_MS) { + await warnStaleAccountingToken(connection, now, lastRefreshMs); + } else { + // Recovered — allow future stale warnings if it goes stale again. + staleWarnedAt.delete(connection.id); + } + } +} + +async function handleExpiredAccountingToken( + accountingService: AccountingService, + connection: AccountingConnection, +): Promise { + if (reauthIncidents.has(connection.id)) { + log("warn", "Accounting token still expired — re-authorization incident already active", { + connectionId: connection.id, + provider: connection.provider, + }); + return; + } + + try { + if (connection.provider === AccountingProvider.XERO) { + await accountingService.refreshXeroToken(connection.id); + } else { + await accountingService.refreshQuickBooksToken(connection.id); + } + log("info", "Auto-healed expired accounting access token", { + connectionId: connection.id, + provider: connection.provider, + }); + } catch (error) { + const now = new Date().toISOString(); + const incident: IncidentRecord = { + subject: `Accounting connection ${connection.id} (${connection.provider}) requires manual re-authorization`, + triggeredAt: now, + dedupeKey: `${dedupPrefix()}-accounting-${connection.id}-reauth`, + }; + + try { + await triggerPagerDutyIncident( + incident, + `[CRITICAL] ${connection.provider} refresh token expired or revoked for connection ${connection.id} — manual re-authorization required`, + { + provider: connection.provider, + connectionId: connection.id, + status: "refresh_failed", + error: toErrorMessage(error), + detectedAt: now, + }, + ); + reauthIncidents.set(connection.id, incident); + log("error", "Accounting re-authorization incident triggered", { + connectionId: connection.id, + provider: connection.provider, + dedupeKey: incident.dedupeKey, + }); + } catch (triggerError) { + log("error", "Failed to trigger re-authorization incident", { + connectionId: connection.id, + reason: toErrorMessage(triggerError), + }); + } + } +} + +async function warnStaleAccountingToken( + connection: AccountingConnection, + now: number, + lastRefreshMs: number, +): Promise { + const lastWarnedAt = staleWarnedAt.get(connection.id); + if (lastWarnedAt && now - lastWarnedAt < staleRealertIntervalMs()) { + return; + } + + const webhookUrls = resolveWarningWebhookUrls(); + if (webhookUrls.length === 0) { + log( + "warn", + "Stale accounting refresh token detected but no alert webhook URL is configured", + { connectionId: connection.id, provider: connection.provider }, + ); + staleWarnedAt.set(connection.id, now); // avoid spamming the logs + return; + } + + const daysSinceRefresh = Math.floor((now - lastRefreshMs) / DAY_MS); + const payload: StaleTokenAlertPayload = { + alertType: "provider_token_stale", + severity: "warning", + generatedAt: new Date().toISOString(), + connections: [ + { + connectionId: connection.id, + provider: connection.provider, + daysSinceRefresh, + refreshTokenLimitDays: refreshTokenLimitDays(connection.provider), + action: + "Reuse or reconnect this integration before the refresh token expires", + }, + ], + }; + + for (const webhookUrl of webhookUrls) { + try { + await postWebhookAlert(webhookUrl, payload); + } catch (error) { + log("error", "Failed to send stale token warning", { + connectionId: connection.id, + webhookUrl, + reason: toErrorMessage(error), + }); + } + } + + staleWarnedAt.set(connection.id, now); + log("warn", "Warned about stale accounting refresh token", { + connectionId: connection.id, + provider: connection.provider, + daysSinceRefresh, + }); +} + +// ─── Main export ────────────────────────────────────────────────────────────── + +/** + * Provider Token Watchdog — runs every 5 minutes via the cron scheduler. + * + * Detects expired/revoked provider credentials (mobile money) and dead or + * dying accounting OAuth tokens before they interrupt service, and alerts via + * PagerDuty (critical) and webhook (warning). + */ +export async function runProviderTokenWatchdogJob(): Promise { + log("info", "Provider token watchdog starting"); + + await checkMobileMoneyCredentials(); + await checkAccountingTokens(); + + log("info", "Provider token watchdog finished", { + activeCredentialIncidents: [...credentialIncidents.keys()], + activeReauthIncidents: [...reauthIncidents.keys()], + }); +} + +// ─── Test-only helpers ──────────────────────────────────────────────────────── +// Prefixed with _ to signal they are not part of the public API. + +/** Returns a snapshot of active mobile-money credential incidents. */ +export function getActiveCredentialIncidents(): ReadonlyMap< + ProviderName, + IncidentRecord +> { + return credentialIncidents; +} + +/** Returns a snapshot of active accounting re-authorization incidents. */ +export function getActiveReauthIncidents(): ReadonlyMap< + string, + IncidentRecord +> { + return reauthIncidents; +} + +/** Clears all tracked incident state — use only in tests. */ +export function _resetWatchdogState(): void { + credentialIncidents.clear(); + reauthIncidents.clear(); + staleWarnedAt.clear(); +} diff --git a/src/jobs/scheduler.ts b/src/jobs/scheduler.ts index 8e6be159..5e61fe42 100644 --- a/src/jobs/scheduler.ts +++ b/src/jobs/scheduler.ts @@ -13,6 +13,7 @@ import { MonitoringService } from "../services/monitoringService"; import { createPagerDutyService } from "../services/pagerDutyService"; import { runProviderBalanceAlertJob } from "./balances"; import { runProviderHealthCheckJob } from "./providerHealthCheck"; +import { runProviderTokenWatchdogJob } from "./providerTokenWatchdog"; import { runKycTierUpgradeJob } from "./kycTierUpgradeJob"; import { runLiquidityRebalanceJob } from "./liquidityRebalanceJob"; import { runCrossChainMonitorJob } from "./crossChainMonitorJob"; @@ -116,6 +117,13 @@ const JOBS: JobConfig[] = [ schedule: process.env.PROVIDER_HEALTH_CHECK_CRON || "*/5 * * * *", handler: runProviderHealthCheckJob, }, + { + name: "provider-token-watchdog", + // Every 5 minutes - detects expired/revoked provider credentials and dead + // or stale accounting OAuth tokens before they interrupt service + schedule: process.env.PROVIDER_TOKEN_WATCHDOG_CRON || "*/5 * * * *", + handler: runProviderTokenWatchdogJob, + }, { name: "provider-reconciliation", // Daily at 4:00 AM - runs automated reconciliation against provider CSV reports diff --git a/src/services/mobilemoney/providers/airtel.ts b/src/services/mobilemoney/providers/airtel.ts index abc5ee2a..a23d323a 100644 --- a/src/services/mobilemoney/providers/airtel.ts +++ b/src/services/mobilemoney/providers/airtel.ts @@ -396,6 +396,40 @@ export class AirtelService { : this.getBalanceViaDirect(); } + /** + * Probes whether the configured credentials/session are still accepted by + * Airtel. In direct mode this fetches a fresh OAuth token; in web mode it + * ensures a valid session (re-logging in if needed); proxy mode is skipped + * because the proxy owns the credentials. Used by the provider token + * watchdog to detect revoked/expired credentials before they interrupt + * service. + */ + async checkAuth(): Promise<{ + success: boolean; + invalidCredentials?: boolean; + error?: unknown; + }> { + try { + if (this.mode === "proxy") { + return { success: true }; + } + + if (this.mode === "web") { + await this.ensureSession(); + return { success: true }; + } + + const token = await this.authenticateDirect(); + return { success: Boolean(token) }; + } catch (error: any) { + return { + success: false, + invalidCredentials: isInvalidCredentialsError(error), + error, + }; + } + } + // ========================================================================= // DIRECT MODE (OAUTH2) // ========================================================================= @@ -1214,3 +1248,19 @@ export class AirtelService { await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); } } + +/** + * Classifies an error as invalid credentials (401/403). Providers using + * `validateStatus: () => true` throw plain Errors with the HTTP status in the + * message (e.g. "Airtel direct auth failed with status 401"), while axios + * rejections carry `error.response.status`. + */ +function isInvalidCredentialsError(error: unknown): boolean { + if (!error) return false; + + const anyError = error as { response?: { status?: number }; message?: string }; + const status = anyError.response?.status; + if (status === 401 || status === 403) return true; + + return /status\s+(401|403)\b/i.test(anyError.message ?? ""); +} diff --git a/src/services/mobilemoney/providers/mtn.ts b/src/services/mobilemoney/providers/mtn.ts index 9877c7a9..2730a12a 100644 --- a/src/services/mobilemoney/providers/mtn.ts +++ b/src/services/mobilemoney/providers/mtn.ts @@ -61,6 +61,29 @@ export class MTNProvider { return token; } + /** + * Probes whether the configured credentials are still accepted by MTN by + * requesting a fresh access token. Used by the provider token watchdog to + * detect revoked/expired API credentials before they interrupt service. + */ + async checkAuth(): Promise<{ + success: boolean; + invalidCredentials?: boolean; + error?: unknown; + }> { + try { + const token = await this.getAccessToken(); + return { success: Boolean(token) }; + } catch (error: any) { + const status = error?.response?.status; + return { + success: false, + invalidCredentials: status === 401 || status === 403, + error, + }; + } + } + async getOperationalBalance() { try { const token = await this.getAccessToken(); diff --git a/src/services/mobilemoney/providers/orange.ts b/src/services/mobilemoney/providers/orange.ts index 338e94fb..8d466db8 100644 --- a/src/services/mobilemoney/providers/orange.ts +++ b/src/services/mobilemoney/providers/orange.ts @@ -619,6 +619,40 @@ export class OrangeProvider { } } + /** + * Probes whether the configured credentials/session are still accepted by + * Orange. In direct mode this fetches a fresh OAuth token; in web mode it + * ensures a valid session (re-logging in if needed); proxy mode is skipped + * because the proxy owns the credentials. Used by the provider token + * watchdog to detect revoked/expired credentials before they interrupt + * service. + */ + async checkAuth(): Promise<{ + success: boolean; + invalidCredentials?: boolean; + error?: unknown; + }> { + try { + if (this.mode === "proxy") { + return { success: true }; + } + + if (this.mode === "web") { + await this.ensureSession(); + return { success: true }; + } + + const token = await this.authenticateDirect(); + return { success: Boolean(token) }; + } catch (error: any) { + return { + success: false, + invalidCredentials: isInvalidCredentialsError(error), + error, + }; + } + } + destroy(): void { this.destroyed = true; if (this.prefetchTimer) { @@ -1111,3 +1145,19 @@ export class OrangeProvider { ); } } + +/** + * Classifies an error as invalid credentials (401/403). Providers using + * `validateStatus: () => true` throw plain Errors with the HTTP status in the + * message (e.g. "Orange direct auth failed with status 401"), while axios + * rejections carry `error.response.status`. + */ +function isInvalidCredentialsError(error: unknown): boolean { + if (!error) return false; + + const anyError = error as { response?: { status?: number }; message?: string }; + const status = anyError.response?.status; + if (status === 401 || status === 403) return true; + + return /status\s+(401|403)\b/i.test(anyError.message ?? ""); +} diff --git a/tests/jobs/providerTokenWatchdog.test.ts b/tests/jobs/providerTokenWatchdog.test.ts new file mode 100644 index 00000000..21d74c4d --- /dev/null +++ b/tests/jobs/providerTokenWatchdog.test.ts @@ -0,0 +1,320 @@ +/** + * tests/jobs/providerTokenWatchdog.test.ts + * + * Tests for the provider token watchdog job — detection of expired/revoked + * mobile money credentials and dead/stale accounting OAuth tokens, with + * PagerDuty (critical) and webhook (warning) alerting. + */ + +import { + runProviderTokenWatchdogJob, + _resetWatchdogState, +} from "../../src/jobs/providerTokenWatchdog"; +import { MTNProvider } from "../../src/services/mobilemoney/providers/mtn"; +import { AirtelService } from "../../src/services/mobilemoney/providers/airtel"; +import { OrangeProvider } from "../../src/services/mobilemoney/providers/orange"; +import { AccountingService } from "../../src/services/accounting"; + +jest.mock("../../src/services/mobilemoney/providers/mtn", () => ({ + MTNProvider: jest.fn(), +})); +jest.mock("../../src/services/mobilemoney/providers/airtel", () => ({ + AirtelService: jest.fn(), +})); +jest.mock("../../src/services/mobilemoney/providers/orange", () => ({ + OrangeProvider: jest.fn(), +})); +jest.mock("../../src/services/accounting", () => ({ + AccountingProvider: { QUICKBOOKS: "quickbooks", XERO: "xero" }, + AccountingService: jest.fn(), +})); + +const MockMTNProvider = MTNProvider as jest.Mock; +const MockAirtelService = AirtelService as jest.Mock; +const MockOrangeProvider = OrangeProvider as jest.Mock; +const MockAccountingService = AccountingService as jest.Mock; + +describe("providerTokenWatchdog", () => { + let mtnCheckAuth: jest.Mock; + let airtelCheckAuth: jest.Mock; + let orangeCheckAuth: jest.Mock; + let accountingServiceMock: { + getAllActiveConnections: jest.Mock; + refreshXeroToken: jest.Mock; + refreshQuickBooksToken: jest.Mock; + }; + + const DAY_MS = 24 * 60 * 60 * 1000; + const now = Date.now(); + + function connection(overrides: Record = {}): any { + return { + id: "conn-1", + provider: "xero", + expiresAt: new Date(now + 3600 * 1000), + updatedAt: new Date(now - 10 * DAY_MS), + ...overrides, + }; + } + + beforeEach(() => { + jest.clearAllMocks(); + _resetWatchdogState(); + + process.env.PAGERDUTY_INTEGRATION_KEY = "test-pagerduty-key"; + delete process.env.PAGERDUTY_DEDUP_KEY; + delete process.env.PROVIDER_TOKEN_ALERT_WEBHOOK_URL; + delete process.env.SLACK_ALERTS_WEBHOOK_URL; + delete process.env.PROVIDER_TOKEN_STALE_REALERT_HOURS; + + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + status: 202, + } as Response); + + mtnCheckAuth = jest.fn().mockResolvedValue({ success: true }); + airtelCheckAuth = jest.fn().mockResolvedValue({ success: true }); + orangeCheckAuth = jest.fn().mockResolvedValue({ success: true }); + + MockMTNProvider.mockImplementation(() => ({ checkAuth: mtnCheckAuth })); + MockAirtelService.mockImplementation(() => ({ checkAuth: airtelCheckAuth })); + MockOrangeProvider.mockImplementation(() => ({ checkAuth: orangeCheckAuth })); + + accountingServiceMock = { + getAllActiveConnections: jest.fn().mockResolvedValue([]), + refreshXeroToken: jest.fn().mockResolvedValue(undefined), + refreshQuickBooksToken: jest.fn().mockResolvedValue(undefined), + }; + MockAccountingService.mockImplementation(() => accountingServiceMock); + }); + + afterEach(() => { + delete process.env.PAGERDUTY_INTEGRATION_KEY; + }); + + describe("mobile money credential probes", () => { + it("probes all three providers and stays silent when credentials are valid", async () => { + await runProviderTokenWatchdogJob(); + + expect(mtnCheckAuth).toHaveBeenCalledTimes(1); + expect(airtelCheckAuth).toHaveBeenCalledTimes(1); + expect(orangeCheckAuth).toHaveBeenCalledTimes(1); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it("triggers a CRITICAL PagerDuty incident when a provider rejects credentials (401/403)", async () => { + mtnCheckAuth.mockResolvedValue({ + success: false, + invalidCredentials: true, + error: new Error("MTN token request failed with status 401"), + }); + + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + const [url, init] = (global.fetch as jest.Mock).mock.calls[0]; + expect(url).toBe("https://events.pagerduty.com/v2/enqueue"); + + const body = JSON.parse(init.body); + expect(body.event_action).toBe("trigger"); + expect(body.dedup_key).toBe( + "proxypay-token-watchdog-mtn-credentials", + ); + expect(body.payload.severity).toBe("critical"); + expect(body.payload.summary).toContain( + "MTN credentials expired or revoked", + ); + expect(body.payload.custom_details.status).toBe("invalid_credentials"); + }); + + it("does not page repeatedly while the credential incident is active", async () => { + mtnCheckAuth.mockResolvedValue({ + success: false, + invalidCredentials: true, + }); + + await runProviderTokenWatchdogJob(); + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + }); + + it("resolves the credential incident once credentials are accepted again", async () => { + mtnCheckAuth.mockResolvedValue({ + success: false, + invalidCredentials: true, + }); + + await runProviderTokenWatchdogJob(); + expect(global.fetch).toHaveBeenCalledTimes(1); + + mtnCheckAuth.mockResolvedValue({ success: true }); + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(2); + const [, init] = (global.fetch as jest.Mock).mock.calls[1]; + const body = JSON.parse(init.body); + expect(body.event_action).toBe("resolve"); + expect(body.dedup_key).toBe( + "proxypay-token-watchdog-mtn-credentials", + ); + }); + + it("ignores unreachable providers (uptime watchdog owns those)", async () => { + mtnCheckAuth.mockResolvedValue({ + success: false, + error: new Error("ECONNREFUSED"), + }); + + await runProviderTokenWatchdogJob(); + + expect(global.fetch).not.toHaveBeenCalled(); + }); + }); + + describe("accounting token checks", () => { + it("does nothing when there are no active connections", async () => { + await runProviderTokenWatchdogJob(); + + expect(accountingServiceMock.getAllActiveConnections).toHaveBeenCalledTimes(1); + expect(accountingServiceMock.refreshXeroToken).not.toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it("auto-heals an expired access token when the refresh succeeds", async () => { + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ provider: "xero", expiresAt: new Date(now - 1000) }), + ]); + + await runProviderTokenWatchdogJob(); + + expect(accountingServiceMock.refreshXeroToken).toHaveBeenCalledWith( + "conn-1", + ); + expect(global.fetch).not.toHaveBeenCalled(); + }); + + it("pages for manual re-authorization when the expired token refresh fails", async () => { + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ provider: "quickbooks", expiresAt: new Date(now - 1000) }), + ]); + accountingServiceMock.refreshQuickBooksToken.mockRejectedValue( + new Error("QuickBooks token refresh failed: invalid_grant"), + ); + + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + const [url, init] = (global.fetch as jest.Mock).mock.calls[0]; + expect(url).toBe("https://events.pagerduty.com/v2/enqueue"); + + const body = JSON.parse(init.body); + expect(body.event_action).toBe("trigger"); + expect(body.dedup_key).toBe( + "proxypay-token-watchdog-accounting-conn-1-reauth", + ); + expect(body.payload.summary).toContain( + "manual re-authorization required", + ); + expect(body.payload.custom_details.error).toContain("invalid_grant"); + }); + + it("stops retrying a connection whose re-authorization incident is active", async () => { + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ expiresAt: new Date(now - 1000) }), + ]); + accountingServiceMock.refreshXeroToken.mockRejectedValue( + new Error("Xero token refresh failed"), + ); + + await runProviderTokenWatchdogJob(); + await runProviderTokenWatchdogJob(); + + expect(accountingServiceMock.refreshXeroToken).toHaveBeenCalledTimes(1); + expect(global.fetch).toHaveBeenCalledTimes(1); + }); + + it("resolves the re-authorization incident once the connection heals", async () => { + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ expiresAt: new Date(now - 1000) }), + ]); + accountingServiceMock.refreshXeroToken.mockRejectedValue( + new Error("Xero token refresh failed"), + ); + + await runProviderTokenWatchdogJob(); + expect(global.fetch).toHaveBeenCalledTimes(1); + + // Connection is reconnected via the OAuth flow → token fresh again + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ expiresAt: new Date(now + 3600 * 1000) }), + ]); + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(2); + const [, init] = (global.fetch as jest.Mock).mock.calls[1]; + const body = JSON.parse(init.body); + expect(body.event_action).toBe("resolve"); + expect(body.dedup_key).toBe( + "proxypay-token-watchdog-accounting-conn-1-reauth", + ); + }); + + it("warns via webhook when a refresh token is approaching inactivity expiry", async () => { + process.env.PROVIDER_TOKEN_ALERT_WEBHOOK_URL = + "https://webhook.example.com/token-alert"; + + // Xero refresh tokens expire after 60 days; updatedAt 50 days ago + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ provider: "xero", updatedAt: new Date(now - 50 * DAY_MS) }), + ]); + + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + const [url, init] = (global.fetch as jest.Mock).mock.calls[0]; + expect(url).toBe("https://webhook.example.com/token-alert"); + + const body = JSON.parse(init.body); + expect(body.alertType).toBe("provider_token_stale"); + expect(body.severity).toBe("warning"); + expect(body.connections[0]).toMatchObject({ + connectionId: "conn-1", + provider: "xero", + daysSinceRefresh: 50, + refreshTokenLimitDays: 60, + }); + }); + + it("does not re-warn about the same stale token within the re-alert interval", async () => { + process.env.PROVIDER_TOKEN_ALERT_WEBHOOK_URL = + "https://webhook.example.com/token-alert"; + + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ updatedAt: new Date(now - 50 * DAY_MS) }), + ]); + + await runProviderTokenWatchdogJob(); + await runProviderTokenWatchdogJob(); + + expect(global.fetch).toHaveBeenCalledTimes(1); + }); + + it("skips the stale-token warning when no webhook is configured", async () => { + const warnSpy = jest.spyOn(console, "warn").mockImplementation(); + + accountingServiceMock.getAllActiveConnections.mockResolvedValue([ + connection({ updatedAt: new Date(now - 50 * DAY_MS) }), + ]); + + await runProviderTokenWatchdogJob(); + + expect(global.fetch).not.toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("no alert webhook URL is configured"), + ); + + warnSpy.mockRestore(); + }); + }); +}); diff --git a/tests/providers/mtn.test.ts b/tests/providers/mtn.test.ts index 743bf2a2..c9c478b9 100644 --- a/tests/providers/mtn.test.ts +++ b/tests/providers/mtn.test.ts @@ -85,6 +85,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: true, data: mockResponse.data, + providerResponseTimeMs: expect.any(Number), }); }); @@ -106,6 +107,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: mockError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -120,6 +122,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: timeoutError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -141,6 +144,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: authError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -155,6 +159,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: networkError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -176,6 +181,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: serverError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -197,6 +203,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: rateLimitError, + providerResponseTimeMs: expect.any(Number), }); }); @@ -338,6 +345,45 @@ describe("MTNProvider", () => { }); }); + describe("checkAuth", () => { + it("reports success when a fresh access token can be obtained", async () => { + mockedAxios.post.mockResolvedValue({ + data: { access_token: "fresh-token", expires_in: 3600 }, + }); + + const result = await provider.checkAuth(); + + expect(result).toEqual({ success: true }); + }); + + it("flags 401/403 responses as invalid credentials", async () => { + mockedAxios.post.mockRejectedValue({ + response: { status: 401 }, + }); + + const result = await provider.checkAuth(); + + expect(result.success).toBe(false); + expect(result.invalidCredentials).toBe(true); + }); + + it("does not flag network/5xx errors as invalid credentials", async () => { + mockedAxios.post.mockRejectedValue(new Error("Network error")); + + const networkResult = await provider.checkAuth(); + expect(networkResult.success).toBe(false); + expect(networkResult.invalidCredentials).toBe(false); + + mockedAxios.post.mockRejectedValue({ + response: { status: 500 }, + }); + + const serverResult = await provider.checkAuth(); + expect(serverResult.success).toBe(false); + expect(serverResult.invalidCredentials).toBe(false); + }); + }); + describe("Status Check", () => { it("should handle successful status check through requestPayment response", async () => { const mockResponse = { @@ -460,6 +506,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: true, data: null, + providerResponseTimeMs: expect.any(Number), }); }); @@ -473,6 +520,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: true, data: undefined, + providerResponseTimeMs: expect.any(Number), }); }); @@ -487,6 +535,7 @@ describe("MTNProvider", () => { expect(result).toEqual({ success: false, error: cancelError, + providerResponseTimeMs: expect.any(Number), }); }); }); diff --git a/tests/services/mobilemoney/orange.test.ts b/tests/services/mobilemoney/orange.test.ts index 9111dada..09bcb21a 100644 --- a/tests/services/mobilemoney/orange.test.ts +++ b/tests/services/mobilemoney/orange.test.ts @@ -205,6 +205,65 @@ describe("OrangeProvider web session flow", () => { removeSession(filePath); }); + describe("checkAuth (direct mode)", () => { + it("reports success when a fresh OAuth token is obtained", async () => { + const directClient = new QueueHttpClient([ + response(200, { access_token: "token-123", expires_in: 3600 }), + ]); + + const provider = new OrangeProvider({ + mode: "direct", + baseUrl: "https://orange.test", + apiKey: "key", + apiSecret: "secret", + directHttpClient: directClient, + clock: () => now, + }); + + await expect(provider.checkAuth()).resolves.toEqual({ success: true }); + }); + + it("flags a 401 from the OAuth token endpoint as invalid credentials", async () => { + const directClient = new QueueHttpClient([ + response(401, { error: "invalid_client" }), + ]); + + const provider = new OrangeProvider({ + mode: "direct", + baseUrl: "https://orange.test", + apiKey: "key", + apiSecret: "secret", + directHttpClient: directClient, + clock: () => now, + }); + + await expect(provider.checkAuth()).resolves.toMatchObject({ + success: false, + invalidCredentials: true, + }); + }); + + it("does not flag network errors as invalid credentials", async () => { + const directClient = new QueueHttpClient([ + response(503, { error: "upstream down" }), + ]); + + const provider = new OrangeProvider({ + mode: "direct", + baseUrl: "https://orange.test", + apiKey: "key", + apiSecret: "secret", + directHttpClient: directClient, + clock: () => now, + }); + + await expect(provider.checkAuth()).resolves.toMatchObject({ + success: false, + invalidCredentials: false, + }); + }); + }); + it("uses a configured proxy without requiring web credentials", async () => { const proxyClient = new QueueHttpClient([ response(202, { transactionId: "proxy-tx" }), diff --git a/tests/services/mobilemoney/providers/airtel.test.ts b/tests/services/mobilemoney/providers/airtel.test.ts index b577955c..5392df36 100644 --- a/tests/services/mobilemoney/providers/airtel.test.ts +++ b/tests/services/mobilemoney/providers/airtel.test.ts @@ -67,4 +67,67 @@ describe("AirtelService", () => { payee: { msisdn: "670000002" }, }); }); + + describe("checkAuth", () => { + function createClient(postImpl: jest.Mock): any { + const client: any = { + post: postImpl, + get: jest.fn(), + }; + mockedAxios.create.mockReturnValue(client as any); + return client; + } + + it("reports success when a fresh OAuth token is obtained", async () => { + const client = createClient(jest.fn()); + client.post.mockImplementation((url: string) => { + if (url === "/auth/oauth2/token") { + return Promise.resolve({ + data: { access_token: "token-123", expires_in: 3600 }, + }); + } + return Promise.reject(new Error(`unexpected request: ${url}`)); + }); + + const service = new AirtelService(); + + await expect(service.checkAuth()).resolves.toEqual({ success: true }); + }); + + it("flags plain-error 401 messages as invalid credentials", async () => { + const client = createClient(jest.fn()); + client.post.mockImplementation((url: string) => { + if (url === "/auth/oauth2/token") { + return Promise.reject( + new Error("Airtel direct auth failed with status 401"), + ); + } + return Promise.reject(new Error(`unexpected request: ${url}`)); + }); + + const service = new AirtelService(); + + await expect(service.checkAuth()).resolves.toMatchObject({ + success: false, + invalidCredentials: true, + }); + }); + + it("does not flag network errors as invalid credentials", async () => { + const client = createClient(jest.fn()); + client.post.mockImplementation((url: string) => { + if (url === "/auth/oauth2/token") { + return Promise.reject(new Error("ECONNREFUSED")); + } + return Promise.reject(new Error(`unexpected request: ${url}`)); + }); + + const service = new AirtelService(); + + await expect(service.checkAuth()).resolves.toMatchObject({ + success: false, + invalidCredentials: false, + }); + }); + }); });