fix: circuit-break nonRetryable price source failures instead of retrying forever

openapi.yaml

@@ -460,6 +460,23 @@ components:
         redis_unavailable:
           type: boolean
           description: Inverse of redis_connected (for legacy monitoring)
+        price_source_circuits:
+          type: array
+          description: Circuit-breaker state for each price source with a nonRetryable failure mode (e.g. an invalid API key)
+          items:
+            type: object
+            properties:
+              source:
+                type: string
+                description: Price source name (e.g. coingecko, coinmarketcap)
+              open:
+                type: boolean
+                description: Whether this source is currently circuit-broken and being skipped
+              openUntil:
+                type: string
+                format: date-time
+                nullable: true
+                description: When the circuit will next allow a retry, or null if closed
       required:
         - status
         - timestamp

src/config.js

@@ -38,6 +38,8 @@ const env = cleanEnv(rawEnv, {
   PRICE_REFRESH_INTERVAL_SECONDS: num({ default: 30 }),
   PRICE_STALE_THRESHOLD_MINUTES: num({ default: 5 }),
   PRICE_ANOMALY_THRESHOLD_PCT: num({ default: 20 }),
+  PRICE_SOURCE_CIRCUIT_COOLDOWN_MS: num({ default: 15 * 60 * 1000 }),
+  PRICE_SOURCE_CIRCUIT_REMINDER_MS: num({ default: 5 * 60 * 1000 }),
   LOG_LEVEL: str({
     default: 'info',
     choices: ['debug', 'info', 'warn', 'error'],
@@ -75,6 +77,15 @@ module.exports = {
     staleThresholdMinutes: env.PRICE_STALE_THRESHOLD_MINUTES,
     anomalyThresholdPercent: env.PRICE_ANOMALY_THRESHOLD_PCT,
   },
+  priceSources: {
+    // How long a source's circuit stays open after a nonRetryable (e.g. 401)
+    // failure before it's attempted again.
+    circuitCooldownMs: env.PRICE_SOURCE_CIRCUIT_COOLDOWN_MS,
+    // Minimum gap between repeated "circuit open, skipping" log lines while
+    // the circuit stays open, so a misconfigured key doesn't spam one log
+    // line per fetch cycle for the entire cooldown window.
+    circuitReminderIntervalMs: env.PRICE_SOURCE_CIRCUIT_REMINDER_MS,
+  },
   auth: {
     adminApiKey: env.ADMIN_API_KEY,
   },

src/index.js

@@ -5,6 +5,7 @@ const helmet = require('helmet');
 const config = require('./config');
 const logger = require('./logger');
 const cache = require('./services/cache');
+const priceOracle = require('./services/priceOracle');
 const priceRefreshJob = require('./jobs/priceRefresh');
 const webhookRetryWorker = require('./jobs/webhookRetryWorker');
 const buildCorsMiddleware = require('./middleware/cors');
@@ -35,6 +36,7 @@ app.get('/health', (req, res) => {
     timestamp: new Date().toISOString(),
     redis_connected: redisConnected,
     redis_unavailable: !redisConnected,
+    price_source_circuits: priceOracle.getSourceCircuitStates(),
   });
 });
 

src/services/priceOracle.js

@@ -9,10 +9,22 @@ const CACHE_PREFIX = 'price:';
 const HISTORY_PREFIX = 'price:history:';
 const SOURCES = [
   { name: 'stellar_dex', fetch: stellarDex.fetchPrice },
-  { name: 'coingecko', fetch: coingecko.fetchPrice },
-  { name: 'coinmarketcap', fetch: coinmarketcap.fetchPrice },
+  { name: 'coingecko', fetch: coingecko.fetchPrice, getCircuitState: coingecko.getCircuitState },
+  { name: 'coinmarketcap', fetch: coinmarketcap.fetchPrice, getCircuitState: coinmarketcap.getCircuitState },
 ];
 
+/**
+ * Circuit-breaker state for every source that has one (currently coingecko
+ * and coinmarketcap — stellar_dex has no API-key/auth failure mode). Lets
+ * callers (e.g. /health) see at a glance which price sources are currently
+ * skipped due to a nonRetryable failure. See #95.
+ */
+function getSourceCircuitStates() {
+  return SOURCES.filter((source) => typeof source.getCircuitState === 'function').map((source) =>
+    source.getCircuitState()
+  );
+}
+
 function median(values) {
   if (values.length === 0) return null;
   const sorted = [...values].sort((a, b) => a - b);
@@ -236,4 +248,5 @@ module.exports = {
   getPrice,
   fetchFreshPrice,
   refreshAllCachedPrices,
+  getSourceCircuitStates,
 };

src/services/sources/circuitBreaker.js

@@ -0,0 +1,70 @@
+'use strict';
+
+const logger = require('../../logger');
+
+/**
+ * A per-source circuit breaker for permanent (nonRetryable) failures like an
+ * invalid/revoked API key. Distinct from ordinary transient failures (network
+ * blips, rate limits): those already self-heal on the next fetch cycle and
+ * are intentionally left untouched by this module.
+ *
+ * State is process-local (module-level, one instance per source per
+ * process) — acceptable because it only affects retry cadence, not
+ * correctness; each horizontally-scaled replica independently rate-limits
+ * its own calls to a known-broken source rather than sharing a single
+ * circuit (see #98 for the analogous cross-replica coordination gap in
+ * scheduled jobs).
+ */
+function createCircuitBreaker({ sourceName, cooldownMs, reminderIntervalMs }) {
+  let openUntil = 0;
+  let lastReminderLoggedAt = 0;
+
+  function isOpen() {
+    return Date.now() < openUntil;
+  }
+
+  /** Call when a fetch is skipped because the circuit is open. Logs at most once per reminderIntervalMs, not once per skipped attempt. */
+  function noteSkipped(context = {}) {
+    const now = Date.now();
+    if (now - lastReminderLoggedAt >= reminderIntervalMs) {
+      logger.warn('Price source circuit open, skipping fetch', {
+        source: sourceName,
+        openUntil: new Date(openUntil).toISOString(),
+        ...context,
+      });
+      lastReminderLoggedAt = now;
+    }
+  }
+
+  /** Call on a nonRetryable failure. Logs distinctly (error level) only the first time the circuit transitions from closed to open. */
+  function open(context = {}) {
+    const wasOpen = isOpen();
+    openUntil = Date.now() + cooldownMs;
+    if (!wasOpen) {
+      logger.error('Price source permanently misconfigured', {
+        source: sourceName,
+        cooldownMs,
+        ...context,
+      });
+      lastReminderLoggedAt = Date.now();
+    }
+  }
+
+  /** Call on a successful fetch. No-op if the circuit was already closed. */
+  function close() {
+    openUntil = 0;
+    lastReminderLoggedAt = 0;
+  }
+
+  function getState() {
+    return {
+      source: sourceName,
+      open: isOpen(),
+      openUntil: openUntil ? new Date(openUntil).toISOString() : null,
+    };
+  }
+
+  return { isOpen, noteSkipped, open, close, getState };
+}
+
+module.exports = { createCircuitBreaker };

src/services/sources/coingecko.js

@@ -1,11 +1,18 @@
 const axios = require('axios');
 const config = require('../../config');
 const logger = require('../../logger');
+const { createCircuitBreaker } = require('./circuitBreaker');
 
 const STELLAR_COINGECKO_MAP = {
   XLM: 'stellar',
 };
 
+const circuit = createCircuitBreaker({
+  sourceName: 'coingecko',
+  cooldownMs: config.priceSources.circuitCooldownMs,
+  reminderIntervalMs: config.priceSources.circuitReminderIntervalMs,
+});
+
 let apiClient = null;
 
 function getClient() {
@@ -30,6 +37,11 @@ async function fetchPrice(assetCode) {
     return null;
   }
 
+  if (circuit.isOpen()) {
+    circuit.noteSkipped({ assetCode });
+    return null;
+  }
+
   try {
     const client = getClient();
     const response = await client.get('/simple/price', {
@@ -39,13 +51,27 @@ async function fetchPrice(assetCode) {
       },
     });
 
+    // A successful HTTP round-trip means any configured API key is valid,
+    // regardless of whether this particular coin had usable price data.
+    circuit.close();
+
     const price = response.data[coinId]?.usd;
     if (price === undefined || price === null) {
       return null;
     }
 
     return price;
   } catch (err) {
+    if (err.response?.status === 401) {
+      // Per CoinGecko's docs, 401 means a missing/invalid API key — a
+      // permanent misconfiguration, not something that self-heals on
+      // retry. Distinct from 403 (CDN/firewall block) and 429 (rate
+      // limit), neither of which indicate a bad key.
+      err.nonRetryable = true;
+      circuit.open({ assetCode });
+      logger.warn('CoinGecko authentication failed', { assetCode });
+      throw err;
+    }
     if (err.response?.status === 429) {
       logger.warn('CoinGecko rate limit hit', { assetCode });
     } else {
@@ -55,4 +81,4 @@ async function fetchPrice(assetCode) {
   }
 }
 
-module.exports = { fetchPrice };
+module.exports = { fetchPrice, getCircuitState: circuit.getState };

src/services/sources/coinmarketcap.js

@@ -1,6 +1,13 @@
 const axios = require('axios');
 const config = require('../../config');
 const logger = require('../../logger');
+const { createCircuitBreaker } = require('./circuitBreaker');
+
+const circuit = createCircuitBreaker({
+  sourceName: 'coinmarketcap',
+  cooldownMs: config.priceSources.circuitCooldownMs,
+  reminderIntervalMs: config.priceSources.circuitReminderIntervalMs,
+});
 
 let apiClient = null;
 
@@ -55,6 +62,11 @@ async function fetchPrice(assetCode, issuer = null) {
     return null;
   }
 
+  if (circuit.isOpen()) {
+    circuit.noteSkipped({ assetCode });
+    return null;
+  }
+
   try {
     const client = getClient();
     const lookupKey = market.id ? String(market.id) : market.symbol;
@@ -65,6 +77,11 @@ async function fetchPrice(assetCode, issuer = null) {
       },
     });
 
+    // A successful HTTP round-trip means the API key is valid, regardless
+    // of whether this particular asset had usable quote data — close the
+    // circuit before evaluating the response shape.
+    circuit.close();
+
     const data = response.data?.data?.[lookupKey];
     if (!data || !data.quote?.USD?.price) {
       return null;
@@ -74,6 +91,7 @@ async function fetchPrice(assetCode, issuer = null) {
   } catch (err) {
     if (err.response?.status === 401) {
       err.nonRetryable = true;
+      circuit.open({ assetCode });
       logger.warn('CoinMarketCap authentication failed', { assetCode });
       throw err;
     }
@@ -89,4 +107,4 @@ async function fetchPrice(assetCode, issuer = null) {
   }
 }
 
-module.exports = { fetchPrice };
+module.exports = { fetchPrice, getCircuitState: circuit.getState };