From f51d1a723ef0e5def35b516909aebb8227491854 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:56:47 -0400 Subject: [PATCH 01/10] Document today's three mechanisms in project memory The canary gate + settlement alarm, the facilitator diagnostics and labels, and the redis CI coverage. Each entry leads with the defect it exists to prevent, because the pattern behind all three was a guarantee that was described but could not fail when violated. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 8f9a448a..43f976b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -332,6 +332,62 @@ with `res.statusCode === 200`. (`node_modules/@x402/express/dist/esm/index.mjs`. which is real but lives on the stdio npm package). Mutation-tested: removing `request_tool` from the listing fails 2 assertions, a fake tool name fails 1, a fake route fails 1. +- **Canary gate + settlement freshness alarm (2026-08-07):** the daily paid canary + stopped buying on **2026-08-02** and reported success every run for five days. Its + gate asked GitHub for the last SUCCESSFUL RUN, but a run whose gate SKIPS the buy + also concludes green, so every skip refreshed the window the next gate read and it + ratcheted permanently shut (measured across 40 runs: not one scheduled run bought + after the gate shipped; every real purchase came from a manual dispatch, which + bypasses the gate via `if: github.event_name == 'schedule'`). Nothing paged, because + skipping is not a failure — the ONLY surface that noticed was `/status`, reporting + the settlement component stale. **The gate now asks PRODUCTION when a canary last + BOUGHT** (`/api/status` settlement observation, written only by a canary that ran), + requiring fresh AND operational; unreachable status or a missing observation proceeds + with the buy, and every `jq` read carries a fallback because jq exits non-zero on a + non-JSON body and `set -e` would fail the gate. The canary job's `if` gained + `!cancelled()`: a job-level `if` with no status function still carries the implicit + `success()` on `needs`, so a FAILED gate would have SKIPPED the buy — the opposite of + what the comment beside it claimed, and never verified. **`heartbeat.yml` now pages on + a stale settlement observation** and self-heals once per episode by dispatching the + canary on FIRST detection only (a dispatch always buys; page rather than loop if + buying is genuinely broken). Proven end-to-end 2026-08-07: alarm fired → dispatched → + found a real failure → opened issues; then the 14:17 UTC SCHEDULED run bought (first + since 08-02) and the recovery branch closed its own issue. `scripts/test-canary-coverage.js` + locks the class: the gate must read `/api/status` and must NOT read `gh run list`, + every jq read must have a fallback, and the `if` must carry a status function. +- **Facilitator failure diagnostics (`src/facilitator-diagnostics.js`, 2026-08-07):** + 15 settle failures across Base/Solana/Polygon/Arbitrum all logged 200 characters of + `Coinbase…` — `@x402/core`'s `responseExcerpt` truncates an + error body at 200 chars, and on an HTML page that budget is spent entirely on markup. + A facilitator outage and an edge REFUSING OUR EGRESS were indistinguishable, and those + need opposite responses (wait vs build the fifth relay — Yahoo/Nasdaq/Sei/Nodely are + the existing four, and Nodely 403s Railway's IP outright). A global-fetch wrapper, + scoped to registered facilitator hosts and non-2xx non-JSON responses only, reads the + body BEFORE the vendor truncates it, strips markup, and classifies: cloudflare + challenge/block, access denied, rate limited, origin error behind the edge, gateway + timeout — keeping `cf-ray`/`server`/`retry-after`. It **clones** before reading + (consuming the body would break settlement), swallows every internal failure, and logs + once at boot so a silent failure to install is visible immediately. **Errors are also + LABELLED with the facilitator that threw them** (`labelFacilitatorErrors`): the failure + hooks log the chain and never the client, so Solana/Polygon/Arbitrum failures read as + Coinbase's words though the boot log routes those to PayAI and only Base to CDP — + clients are tried in order, so the surfacing error is the FIRST tried, not the chain's + owner. The label is **PREFIXED, never substituted**: `isPreBroadcastSettleRejection` + matches `settle failed (402)` as a substring, so replacing the message would silently + break the fallback's safety classification. `scripts/test-facilitator-diagnostics.js` + (30 assertions, offline, in CI). +- **Redis has REAL coverage in CI (2026-08-07):** nothing had ever connected to a redis. + `test-shared-limit.js` injects a fake store on purpose (it proves "two callers share + one counter", and a fake proves that exactly), which left the CLIENT path untested — + so a redis 4→6 bump arrived with a green CI that could not have caught a client + regression, the same worthless green as the tesseract 5→7 trap. Prod is **NOT** + in-memory (verified against Railway: `REDIS_URL` and `RATE_LIMIT_REPLICAS` are set, + and the shared limiter FAILS CLOSED). The test job now runs a `redis:7-alpine` service + container and `scripts/test-redis-integration.js` drives the real client (cap-of-1, + over-limit decrement, refund flooring, cache round trip). It asserts `degraded === false` + so it cannot pass via the fail-closed path with no server, and it **exits 1 rather than + skipping** when `REDIS_URL` is absent — a skipped integration test is why this went + untested at all. - **Marketplace latency / snapshot caching (`src/x402-economy.js`):** `GET /marketplace` (and `/api/x402-economy`) render from `x402EconomySnapshot()` — a ~500ms on-chain read (EIP-3009 USDC settlements on Base via CDP SQL). It is **stale-while-revalidate**: a fresh From e2bb5c7c617b0cceb51249c3ffa624b7bd990846 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:15:11 -0400 Subject: [PATCH 02/10] [test][deploy] Learn a seller's price from the live 402 when the catalogue has none A seller listed 39 endpoints and every row indexed as price:null, priceUsd:0, payable:"unknown" - while each endpoint returns a textbook x402 v2 challenge (eip155:8453, Base USDC, amount 990000 = $0.99, real payTo) the moment you POST {} at it. Confirmed independently before touching anything: their manifest, their live 402, and our index rows. Two causes, both ours, and neither specific to them: 1. A manifest may list `resources` as bare URL STRINGS - theirs does, and the shape is permitted. normaliseManifestTools reads a price only from an OBJECT, so a string-listing seller is permanently priceless however well their endpoints behave. It also leaves method defaulting to GET, and their routes 404 on GET and 402 on POST, so a GET-only probe sees a dead catalogue. 2. probePaywall - the only thing that talks to a seller's endpoint - filters on `Number(t.price) > 0`. A priceless route is never probed, and probing is the only thing that would give it a price. Circular by construction: the sellers who most need the probe are precisely the ones excluded from it. Not one seller's problem. Measured across the index the same day: 146 of 500 sellers had ZERO priced rows and 127 rows read payable:"unknown". OpenAPI cannot close this - it has no place for an x402 quote - so the 402 itself is the only source of truth, which is why the router already reads a live 402 for payTo before spending. This reads the same challenge for price and networks, and mirrors bazaarItemToTool's preference order (Base USDC, then any USDC, then first) so a live-probed row and a Bazaar row stay comparable. Conservative about money in three ways. An asset we cannot price leaves price NULL and still records the networks, so the row reads payable-on-Base rather than a guessed figure - $0 would publish a paid tool as free, the one wrong answer a buyer acts on immediately. PUT/PATCH/DELETE are never probed, so an unpaid probe cannot mutate a stranger's server. And it only ever ADDS: a priced row is skipped and a failed probe leaves the row untouched. Gentle on sellers, per the #645 lesson: at most 3 priceless routes per seller per crawl, per-ROUTE backoff through probeDue, and a route that gets priced is never a candidate again. Verified against the reporting seller's live endpoint through the real code path - assertPublicUrl and the SSRF dispatcher included - GET 404 skipped, POST 402 read, $0.99 and their payTo learned. 28 offline assertions on fixtures captured from that same response. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/deploy.yml | 3 + scripts/test-x402-live-quote.js | 110 ++++++++++++++++++++++ src/x402-index.js | 83 +++++++++++++++++ src/x402-live-quote.js | 158 ++++++++++++++++++++++++++++++++ 4 files changed, 354 insertions(+) create mode 100644 scripts/test-x402-live-quote.js create mode 100644 src/x402-live-quote.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 021016f1..85a529fe 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1012,6 +1012,9 @@ jobs: - name: Boot /supported guard (a dead facilitator costs ONE rail, not every paid route — the 2026-08-01 Celo outage; probe-driven drop, fail-open on total blindness, escape hatch — offline) run: node scripts/test-supported-guard.js + - name: "Live 402 quote (a priceless row learns its price from the challenge; unpriceable never becomes $0; PUT/PATCH/DELETE never probed — offline)" + run: node scripts/test-x402-live-quote.js + - name: "Facilitator diagnostics (an HTML error page must name itself: block vs rate limit vs origin failure, and never touch the response — offline)" run: node scripts/test-facilitator-diagnostics.js diff --git a/scripts/test-x402-live-quote.js b/scripts/test-x402-live-quote.js new file mode 100644 index 00000000..f5876f8e --- /dev/null +++ b/scripts/test-x402-live-quote.js @@ -0,0 +1,110 @@ +// Offline tests for src/x402-live-quote.js. +// +// Fixtures are the REAL shapes, captured from a live seller on 2026-08-07 who +// reported 39 endpoints indexed at price:null while every one of them returns a +// textbook 402. If these assertions pass against anything less than that real +// challenge, they are not testing what went wrong. +import { acceptsFromLive402, quoteFromAccepts, probeMethodsFor, isQuoteResponse } from "../src/x402-live-quote.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; + +// The accepts entry exactly as the reporting seller's 402 carries it: $0.99 as +// 990000 atomic units of Base USDC, with the name in `extra`. +const REAL_ACCEPT = { + scheme: "exact", + network: "eip155:8453", + asset: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + amount: "990000", + payTo: "0x0bac88e8B47D9F2dC38E66dB9dA4b41032d24065", + maxTimeoutSeconds: 60, + extra: { name: "USDC", version: "2" }, +}; +const HEADER = Buffer.from(JSON.stringify({ x402Version: 2, accepts: [REAL_ACCEPT] })).toString("base64"); + +// --- reading the challenge --------------------------------------------------- +{ + ok(acceptsFromLive402({ header: HEADER })?.[0]?.amount === "990000", + "accepts are read from the base64 payment-required HEADER (x402 v2's home for them)"); + + // The same seller ALSO nests it in the body under `payment`. A reader that + // only understood a top-level `accepts` would have missed it. + const body = JSON.stringify({ error: { code: "PAYMENT_REQUIRED" }, payment: { rail: "cdp_x402", accepts: [REAL_ACCEPT] } }); + ok(acceptsFromLive402({ body })?.[0]?.payTo === REAL_ACCEPT.payTo, + "accepts are read from a body that NESTS them under `payment`"); + + ok(acceptsFromLive402({ body: JSON.stringify({ accepts: [REAL_ACCEPT] }) })?.length === 1, + "a plain top-level accepts body still works"); + + ok(acceptsFromLive402({ header: "not-base64-at-all", body: JSON.stringify({ accepts: [REAL_ACCEPT] }) })?.length === 1, + "an undecodable header falls through to the body rather than giving up"); + + ok(acceptsFromLive402({ header: "", body: "nope" }) === null, + "an unreadable challenge is null - never an empty-but-truthy quote"); + ok(acceptsFromLive402({}) === null, "nothing in, null out"); +} + +// --- pricing ----------------------------------------------------------------- +{ + const q = quoteFromAccepts([REAL_ACCEPT]); + ok(q.price === 0.99, `990000 atomic USDC prices as $0.99, not 990000 (got ${q.price})`); + ok(q.networks.includes("eip155:8453"), "the network travels with the quote"); + ok(q.payTo === REAL_ACCEPT.payTo, "the payTo is captured so the router can check where money goes"); + + // Base USDC wins over another chain, matching bazaarItemToTool's order, so a + // live-probed row and a Bazaar row are comparable. + const multi = quoteFromAccepts([ + { network: "solana:x", amount: "5000000", payTo: "sol", extra: { name: "USDC" } }, + REAL_ACCEPT, + ]); + ok(multi.price === 0.99 && multi.network === "eip155:8453", + `Base USDC is preferred when several chains are offered (got ${multi.price} on ${multi.network})`); + ok(multi.networks.length === 2, "every offered chain is still recorded"); +} + +{ + // THE MONEY ASSERTION. An asset we cannot price must not become $0 - that + // would publish a paid tool as free, and "free" is the one wrong answer a + // buyer acts on immediately. + const unknown = quoteFromAccepts([{ network: "eip155:8453", amount: "12345", payTo: "0xabc", extra: { name: "WEIRDTOKEN" } }]); + ok(unknown.price === null, `an unpriceable asset yields null, never 0 (got ${unknown.price})`); + ok(unknown.networks.includes("eip155:8453"), + "…but the network is still recorded, so the row reads payable-on-Base rather than unknown"); + + const explicit = quoteFromAccepts([{ network: "eip155:1", amount: "1500000000000000000", payTo: "0x1", extra: { name: "DAI", decimals: 18 } }]); + ok(explicit.price === 1.5, `an explicit decimals hint is honoured (got ${explicit.price})`); + + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "-5", extra: { name: "USDC" } }]).price === null, + "a negative amount is corrupt, not free"); + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "abc", extra: { name: "USDC" } }]).price === null, + "a non-numeric amount does not become NaN dollars"); + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "0", extra: { name: "USDC" } }]).price === 0, + "an explicit zero IS free and is reported as such"); + ok(quoteFromAccepts([]) === null && quoteFromAccepts(null) === null, "no accepts, no quote"); +} + +// --- which methods to probe -------------------------------------------------- +{ + // The defect in one assertion: the reporting seller's routes 404 on GET and + // 402 on POST. A GET-only prober sees a dead catalogue. + ok(probeMethodsFor({ method: "GET", methodInferred: true }).includes("POST"), + "an INFERRED GET still tries POST - the seller whose routes 404 on GET and 402 on POST"); + ok(probeMethodsFor({}).join(",") === "GET,POST", "no stated method tries both, GET first (cheapest)"); + ok(probeMethodsFor({ method: "POST" }).join(",") === "POST", "a stated POST is taken at its word"); + ok(probeMethodsFor({ method: "GET", methodInferred: false }).join(",") === "GET,POST", + "a stated GET still falls back to POST, because a 404 costs one request and a missed catalogue costs a seller"); + + // An unpaid probe must never be able to mutate a stranger's server. + for (const m of ["PUT", "PATCH", "DELETE"]) { + ok(probeMethodsFor({ method: m }).length === 0, `${m} is never probed - an unpaid probe must not mutate anything`); + } +} + +{ + ok(isQuoteResponse(402) === true, "402 is the healthy answer to an unpaid call"); + ok(isQuoteResponse(200) === false, "200 means the route is not paywalled, which is not a quote"); + ok(isQuoteResponse(404) === false && isQuoteResponse(500) === false, "errors are not quotes"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/src/x402-index.js b/src/x402-index.js index f0224761..baeed937 100644 --- a/src/x402-index.js +++ b/src/x402-index.js @@ -36,6 +36,7 @@ import { fetchAllBazaarItems, isBazaarDiscoveryUrl } from "./bazaar-pager.js"; import { RAILS, railKey, truncateCaip2 } from "./rails.js"; import { CHAIN_PAGES, marketSellers } from "./market-page.js"; import { WELL_KNOWN_PATH, discoveryNote } from "./discovery-note.js"; +import { acceptsFromLive402, quoteFromAccepts, probeMethodsFor, isQuoteResponse } from "./x402-live-quote.js"; import { summarize, fmtUsd, fmtPct } from "./economy.js"; import { rankBy, canonicalHost } from "./leaderboard.js"; import { routeExecuteHint } from "./tools/route-execute.js"; @@ -1144,6 +1145,81 @@ function paywallProbeDue() { return paywallProbeCursor++ % Math.max(1, Math.ceil(cache.size / PAYWALL_PROBES_PER_CYCLE) || 1) === 0; } +// How many priceless routes we will quote-probe per seller per crawl. The +// crawl runs every 5 minutes across ~2,200 origins, so this is the difference +// between "we learn a catalogue's prices within the hour" and "we hammer a +// stranger's server". A route that gets priced is never probed again (it has a +// price); one that cannot be priced backs off through probeDue like every +// other path. See the #645 note below on why per-PATH backoff matters. +const LIVE_QUOTE_PROBES_PER_CRAWL = 3; + +/** + * Learn price + networks from a live 402 for rows that have neither. + * + * THE DEFECT (reported by a seller, 2026-08-07): a manifest may list + * `resources` as bare URL strings, which carry no price, and probePaywall - + * the only thing that talks to a seller's endpoint - filters on + * `Number(t.price) > 0`. So a priceless row was never probed, and probing is + * the only thing that could have given it a price. Their 39 endpoints indexed + * at price:null while every one returned a textbook 402 on POST. Across the + * index that same day: 146 of 500 sellers had zero priced rows. + * + * Only ever ADDS information: a row that already has a price is skipped, and a + * probe that cannot produce a quote leaves the row exactly as it was. + */ +async function enrichLiveQuotes(tools, originUrl) { + if (!Array.isArray(tools) || !tools.length) return tools; + const candidates = tools.filter( + (t) => t + && typeof t.route === "string" && t.route.startsWith("/") + && t.seller !== LOCAL_SELLER // never probe ourselves + && !(Number(t.price) > 0) // already priced: nothing to learn + && !(Array.isArray(t.networks) && t.networks.length) // already payable-evidenced + && probeMethodsFor(t).length // never PUT/PATCH/DELETE + && probeDue(originUrl, `quote:${t.route}`), + ).slice(0, LIVE_QUOTE_PROBES_PER_CRAWL); + if (!candidates.length) return tools; + + const { assertPublicUrl, ssrfDispatcher } = await import("./tools/fetch-guard.js"); + for (const tool of candidates) { + const target = `${originUrl}${tool.route}`; + let learned = null; + for (const method of probeMethodsFor(tool)) { + try { + // Crawled URLs are external data and could DNS-rebind between crawl and + // now: validate then pin, exactly as probePaywall does. + await assertPublicUrl(target); + const res = await fetch(target, { + method, + headers: { Accept: "application/json", ...(method === "POST" ? { "Content-Type": "application/json" } : {}) }, + ...(method === "POST" ? { body: "{}" } : {}), + dispatcher: ssrfDispatcher, + redirect: "manual", + signal: AbortSignal.timeout(8000), + }); + if (!isQuoteResponse(res.status)) continue; // 404 on GET is expected for a POST-only seller + // The quote lives in the header for x402 v2 and in the body for several + // real sellers; read a bounded slice of both and let the parser decide. + const body = await res.text().catch(() => ""); + const quote = quoteFromAccepts( + acceptsFromLive402({ header: res.headers.get("payment-required"), body: body.slice(0, 64_000) }), + ); + if (quote) { learned = { ...quote, method }; break; } + } catch { /* unreachable, blocked, or malformed - try the next method */ } + } + noteProbeOutcome(originUrl, `quote:${tool.route}`, Boolean(learned)); + if (!learned) continue; + // Price may be null for an asset we refuse to guess at; the networks alone + // still move the row from payable:"unknown" to payable:"x402", which is the + // honest and useful half of the answer. + if (learned.price != null && !(Number(tool.price) > 0)) tool.price = learned.price; + if (learned.networks?.length) tool.networks = [...new Set([...(tool.networks || []), ...learned.networks])]; + if (learned.method && learned.method !== tool.method) { tool.method = learned.method; tool.methodInferred = false; } + tool.quoteSource = "live-402"; + } + return tools; +} + async function probePaywall(tools) { // A cached tool row has NO `url` field — the callable URL is derived as // seller + route, the same way routeQuery builds it (see the `url:` mapping @@ -1308,6 +1384,9 @@ async function crawlSeller(originUrl) { // mergeManifestIntoTools for the 16 -> 30 regression that proved why. tools = mergeManifestIntoTools(normaliseManifestTools(manifest, originUrl), tools); tools = dropUnvouchedNonProductRoutes(tools, (bazaarToolsByOrigin.get(originUrl) || []).map((t) => t.route)); + // Learn prices the catalogue could not carry. Bounded per seller per crawl + // and backed off per route; only ever adds information. + tools = await enrichLiveQuotes(tools, originUrl); cache.set(originUrl, { manifest, @@ -1409,6 +1488,10 @@ async function crawlSeller(originUrl) { bazaarTools.map((t) => t.route) ); if (tools.length) { + // Same enrichment as the manifest path. A seller discovered through the + // FALLBACK surfaces is even less likely to have published a price, so + // skipping it here would leave the worst-served sellers unpriced. + await enrichLiveQuotes(tools, originUrl); // A real (non-synthesized) manifest from a past crawl is kept; a stale // synthesized one is rebuilt so a newly appeared openapi title wins. const keepManifest = prev?.manifest && !prev.manifest.synthesized ? prev.manifest : null; diff --git a/src/x402-live-quote.js b/src/x402-live-quote.js new file mode 100644 index 00000000..1fe038e2 --- /dev/null +++ b/src/x402-live-quote.js @@ -0,0 +1,158 @@ +// Learn a seller's price from the only surface guaranteed to have it: a live 402. +// +// WHY THIS EXISTS (2026-08-07, reported by a seller who had just listed). +// Canu Verify listed 39 endpoints and every row came back price:null, +// priceUsd:0, payable:"unknown" - while each endpoint returns a textbook x402 v2 +// challenge (eip155:8453, Base USDC, amount 990000 = $0.99, real payTo) the +// moment you POST `{}` at it. Measured across the index the same day: 146 of +// 500 sellers had ZERO priced rows and 127 rows read payable:"unknown". So this +// was never one seller's problem. +// +// Two causes, both ours: +// 1. A manifest may list `resources` as bare URL STRINGS (theirs does, and +// the spec permits it). normaliseManifestTools can only read a price from +// an OBJECT, so a string-listing seller is permanently priceless no matter +// how well their endpoints behave. +// 2. probePaywall - the one thing that talks to a seller's endpoint - filters +// on `Number(t.price) > 0`. A route with no price is never probed, and +// probing is the only thing that would give it one. Circular by +// construction: the sellers who most need the probe are the only ones +// excluded from it. +// +// OpenAPI cannot close this: it has no place for an x402 quote. The 402 itself +// is the source of truth, which is also why the router already reads a live 402 +// for payTo (payToFromLive402) before spending. This reads the same challenge +// for price and networks. +// +// Deliberately CONSERVATIVE about money: an amount we cannot price leaves the +// price null and still records the networks, so the row becomes "payable over +// x402 on Base" rather than a guessed dollar figure. Under-claiming is the +// safe direction when the number decides what a buyer is charged. + +/** USDC is 6 decimals on every chain we accept. Anything else we refuse to + * price rather than guess - a wrong exponent is a 1000x pricing error. */ +const USDC_DECIMALS = 6; +const USDC_NAME = /^(usdc|usd coin)$/i; + +/** + * Pull the accepts array out of a live 402. + * + * x402 v2 carries it base64 in the `payment-required` HEADER with an empty (or + * unrelated) body; other sellers put it in the JSON body; Canu does BOTH, with + * the body nesting it under `payment`. All three are read, header first, + * because the header is the spec's home for it. + */ +export function acceptsFromLive402({ header, body } = {}) { + const dig = (obj) => { + if (!obj || typeof obj !== "object") return null; + if (Array.isArray(obj.accepts) && obj.accepts.length) return obj.accepts; + // Sellers wrap the envelope: { payment: { accepts } }, { x402: { accepts } }. + for (const k of ["payment", "x402", "paymentRequired", "payment_required", "data"]) { + const nested = obj[k]; + if (nested && typeof nested === "object" && Array.isArray(nested.accepts) && nested.accepts.length) { + return nested.accepts; + } + } + return null; + }; + + if (typeof header === "string" && header.trim()) { + try { + const decoded = JSON.parse(Buffer.from(header.trim(), "base64").toString("utf8")); + const hit = dig(decoded); + if (hit) return hit; + } catch { /* fall through to the body */ } + } + if (typeof body === "string" && body.trim()) { + try { + const hit = dig(JSON.parse(body)); + if (hit) return hit; + } catch { /* unreadable */ } + } + if (body && typeof body === "object") { + const hit = dig(body); + if (hit) return hit; + } + return null; +} + +/** Is this accepts entry denominated in USDC? Checked by NAME (what x402 v2 + * puts in `extra`) rather than by address, so a new chain's USDC works without + * a table to forget to update. */ +function isUsdc(a) { + return USDC_NAME.test(String(a?.extra?.name || "").trim()); +} + +/** + * Turn a live 402's accepts into the fields the index stores. + * + * Mirrors bazaarItemToTool's preference order deliberately - prefer Base USDC, + * then any USDC, then the first entry - so a row learned from a live probe and + * a row learned from the Bazaar are directly comparable. Two price ladders for + * the same catalogue would be worse than none. + * + * Returns price:null (never 0, never a guess) when the amount cannot be priced, + * while still returning the networks, because "payable on Base, amount unknown" + * is both true and useful, and 0 would read as free. + */ +export function quoteFromAccepts(accepts) { + const list = Array.isArray(accepts) ? accepts.filter((a) => a && typeof a === "object") : []; + if (!list.length) return null; + + const preferred = + list.find((a) => a.network === "eip155:8453" && isUsdc(a)) || + list.find(isUsdc) || + list[0]; + + let price = null; + const decimals = Number.isInteger(a$(preferred?.extra?.decimals)) ? a$(preferred.extra.decimals) + : isUsdc(preferred) ? USDC_DECIMALS + : null; + if (decimals != null && preferred?.amount != null) { + const n = Number(preferred.amount); + // A negative or non-finite amount is corrupt, not free. + if (Number.isFinite(n) && n >= 0) price = n / 10 ** decimals; + } + + return { + price, + networks: [...new Set(list.map((a) => a.network).filter((n) => typeof n === "string" && n))], + payTo: typeof preferred?.payTo === "string" ? preferred.payTo : null, + asset: typeof preferred?.asset === "string" ? preferred.asset : null, + // Which entry priced it, so a surprising number can be traced to its source. + network: typeof preferred?.network === "string" ? preferred.network : null, + }; +} + +/** Number() that refuses strings-that-are-not-numbers, for the decimals hint. */ +function a$(v) { + if (typeof v === "number") return v; + if (typeof v === "string" && /^\d+$/.test(v.trim())) return Number(v.trim()); + return NaN; +} + +/** + * Which HTTP methods to try, in order, for a route whose price we do not know. + * + * A GET-only prober cannot see a POST-only seller: Canu's endpoints 404 on GET + * and 402 on POST, so the whole catalogue read as priceless. When the catalogue + * states a method we trust it and try only that; when the method was INFERRED + * (or absent) we try GET then POST, because a POST with `{}` to a GET endpoint + * is harmless and a GET to a POST endpoint is a 404 that costs one request. + * + * Never PUT/PATCH/DELETE: an unpaid probe must not be able to mutate anything, + * even by accident, on a stranger's server. + */ +export function probeMethodsFor(tool) { + const stated = String(tool?.method || "").toUpperCase(); + if (stated === "POST") return ["POST"]; + if (stated === "GET" && tool?.methodInferred !== true) return ["GET", "POST"]; + if (stated && stated !== "GET" && stated !== "POST") return []; + return ["GET", "POST"]; +} + +/** Is this response a usable x402 quote? 402 is the only healthy answer to an + * unpaid call; a 200 means the route is not paywalled at all. */ +export function isQuoteResponse(status) { + return status === 402; +} From c6355e934c603b6d219976b2170014ec0296219b Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:15:11 -0400 Subject: [PATCH 03/10] [test][deploy] Trigger CI for the live-402 quote enrichment Co-Authored-By: Claude Opus 5 (1M context) --- .github/trigger-deploy | 2 +- .github/trigger-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 07da5bc0..13c6292f 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -1 +1 @@ -1786112186 +1786115711 diff --git a/.github/trigger-test b/.github/trigger-test index aee45a80..13c6292f 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786111133 +1786115711 From c16338ebc8d612f83635e6b6bf9a14e7b1db2990 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:30:24 -0400 Subject: [PATCH 04/10] [test][deploy] Bound what one buyer can make us spend, then raise the routing ceiling Two things, and the order matters: the guard is what makes the price rise safe. THE HOLE. @x402/express runs the handler FIRST and settles AFTER, and the external routing handler pays a third-party seller from our spending wallet. So: buyer's payment verifies, we pay the seller real USDC, our settlement fails, buyer is charged nothing, we are out the spend. Self-dealt - one wallet listing the seller and buying from it - every drained dollar lands back in the attacker's pocket, bounded per call only by the tier cap. Verify-then-fail-to-settle is not theoretical; it happens naturally when a payer's balance drops between the two, which is documented on Solana where our own best buyer drained to $0 and its last four purchases "timed out". What already existed bounds WHAT we pay and none of it bounds WHETHER WE GET PAID: the canonical-USDC asset pin (a decoy in another token cannot be signed), the tier cap re-checked against the live 402 rather than the seller's advertised price, and the 50-settlement/3-payer reliability floor. All of them run before a spend that settlement has not yet blessed. So a payer now carries a DEBT CEILING of unsettled upstream spend. Recorded before the buy, resolved on the FINAL response - res.on("finish") with statusCode 200, after settlement - never on handler success, which is precisely the state that precedes a settlement failure. An unsettled spend keeps counting until it ages out; a settled one clears instantly. It only ever bites a wallet whose payments are failing. Not a reputation system: 25 settled calls in a row are never impeded. THE PRICE. The $0.50 underlying ceiling made the premium half of the index unroutable - the seller who reported the price:null bug prices their gates at $0.99, $1.50 and $2.99, every one above the top tier, so the router could only 409 them to their own direct route. route-execute-pro covers underlying <=$3.00 at $3.30, the same 10% spread as the max tier. Three bugs found while doing it, two of them mine: - The first draft registered res.on("finish") inside the tool handler, where `res` does not exist: handlers are called as handler(input, req). It would have installed nothing, reported nothing, and left the guard recording spends that never resolved. The handle now rides the request and server.js resolves it centrally. - I added a tier without adding it to SELF_FUNDING_SLUGS, so its revenue would have settled to the treasury while its spend came from the burner - a slow one-way drain. Caught by the existing test that locks that set against EXEC_TIERS, written after the same mistake in July. - The ceiling and the largest tier are coupled: a ceiling below the biggest underlying cap makes that tier dead on arrival, refusing every payer including honest ones, with nothing to report it. Now asserted, not merely commented. 20 + 45 offline assertions. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/deploy.yml | 3 + scripts/test-external-spend-guard.js | 147 +++++++++++++++++++++++++ scripts/test-route-execute.js | 13 ++- src/external-spend-guard.js | 157 +++++++++++++++++++++++++++ src/payments.js | 4 +- src/server.js | 16 +++ src/tools/route-execute.js | 38 ++++++- 7 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 scripts/test-external-spend-guard.js create mode 100644 src/external-spend-guard.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 85a529fe..779d5180 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1015,6 +1015,9 @@ jobs: - name: "Live 402 quote (a priceless row learns its price from the challenge; unpriceable never becomes $0; PUT/PATCH/DELETE never probed — offline)" run: node scripts/test-x402-live-quote.js + - name: "External spend guard (per-payer debt ceiling: an UNSETTLED upstream spend keeps counting, so verify-then-fail-to-settle cannot drain the wallet — offline)" + run: node scripts/test-external-spend-guard.js + - name: "Facilitator diagnostics (an HTML error page must name itself: block vs rate limit vs origin failure, and never touch the response — offline)" run: node scripts/test-facilitator-diagnostics.js diff --git a/scripts/test-external-spend-guard.js b/scripts/test-external-spend-guard.js new file mode 100644 index 00000000..75ae0192 --- /dev/null +++ b/scripts/test-external-spend-guard.js @@ -0,0 +1,147 @@ +// Offline tests for src/external-spend-guard.js. +// +// The hole it closes: settlement runs AFTER the handler, and the external +// routing handler pays a third-party seller from our wallet. A payment that +// VERIFIES and then fails to SETTLE leaves us out the upstream spend with the +// buyer charged nothing. Self-dealt - one wallet listing the seller and buying +// from it - every drained dollar returns to the attacker. +import { + maySpend, noteSpend, resolveSpend, payerExposureUsd, exposureSnapshot, __reset, +} from "../src/external-spend-guard.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; + +const A = "0xAbCdEf0123456789AbCdEf0123456789AbCdEf01"; + +// --- the attack, in one block ------------------------------------------------ +{ + __reset(); + // An explicit ceiling, not the shipped default: this block is about the + // MECHANISM, and tying it to whatever the default happens to be today would + // make it pass for the wrong reason the next time a tier moves. + const CEIL = { maxUnsettledUsd: 0.75 }; + // Call 1: allowed, we spend upstream. + ok(maySpend(A, 0.5, CEIL).ok, "a fresh payer may spend"); + const h1 = noteSpend(A, 0.5); + ok(payerExposureUsd(A) === 0.5, "the spend counts as exposure while unresolved"); + + // Their payment FAILS to settle. This is the whole point: handler success is + // not revenue, and the exposure must survive it. + resolveSpend(h1, false); + ok(payerExposureUsd(A) === 0.5, + `an UNSETTLED spend keeps counting against the payer (got ${payerExposureUsd(A)})`); + + // Call 2 from the same wallet is refused before we spend a second time. + const second = maySpend(A, 0.5, CEIL); + ok(second.ok === false, "a second call is refused while the first is unpaid - the drain stops at one"); + ok(/has not settled/i.test(second.reason), `the refusal explains itself (got: ${second.reason})`); +} + +{ + // The honest buyer's path: settle, and exposure clears immediately. + __reset(); + const h = noteSpend(A, 0.5); + resolveSpend(h, true); + ok(payerExposureUsd(A) === 0, "a SETTLED spend clears the exposure at once"); + ok(maySpend(A, 0.5, { maxUnsettledUsd: 0.75 }).ok, "and the payer may immediately spend again - this is a debt ceiling, not a reputation"); +} + +{ + // A wallet that pays reliably is never impeded, however many calls it makes. + __reset(); + let everRefused = false; + for (let i = 0; i < 25; i++) { + if (!maySpend(A, 0.5).ok) everRefused = true; + resolveSpend(noteSpend(A, 0.5), true); + } + ok(!everRefused && payerExposureUsd(A) === 0 && maySpend(A, 0.5).ok, + "25 settled calls in a row are never refused and leave zero exposure - a good buyer never hits the ceiling"); +} + +// --- identity handling ------------------------------------------------------- +{ + __reset(); + const h = noteSpend(A.toLowerCase(), 0.5); + resolveSpend(h, false); + ok(payerExposureUsd(A.toUpperCase().replace("0X", "0x")) === 0.5, + "EVM addresses are case-insensitive, so a payer cannot reset their ledger by changing case"); + + // base58 / Stellar / Algorand are case-SENSITIVE: folding them merges + // distinct payers, the same rule src/payer.js enforces. + const s1 = "GDNJXCKW7ZM7GEEVP674TWPU26YJNBQ2FI4ZIPRKTPTNUEJMDHFJWWRL"; + const h2 = noteSpend(s1, 0.4); + resolveSpend(h2, false); + ok(payerExposureUsd(s1.toLowerCase()) === 0, + "a base58/Stellar address is NOT case-folded - folding would merge distinct payers"); +} + +{ + __reset(); + // An unattributable payer (free mode, a rail whose payer we cannot read) is + // allowed: refusing would break every legitimate buyer on those rails, and a + // single call is still bounded by the tier cap. + const v = maySpend(null, 0.5); + ok(v.ok === true && /not attributable/i.test(v.reason), + "an unreadable payer is allowed and says why - the tier cap still bounds the call"); + ok(noteSpend(null, 0.5) === null, "…and nothing is recorded for a payer we cannot name"); +} + +// --- ceiling arithmetic ------------------------------------------------------ +{ + __reset(); + ok(maySpend(A, 0.5, { maxUnsettledUsd: 0.5 }).ok, "exactly at the ceiling is allowed"); + const h = noteSpend(A, 0.5); + resolveSpend(h, false); + ok(maySpend(A, 0.01, { maxUnsettledUsd: 0.5 }).ok === false, + "one cent past the ceiling is refused - the check is on the TOTAL, not the single call"); +} + +{ + // An unresolved row must not bar a payer forever (a process restart, a + // response that never finished), but must not clear so fast a loop outruns it. + __reset(); + const now = 1_000_000; + noteSpend(A, 0.5, now); + ok(payerExposureUsd(A, now + 60_000) === 0.5, "exposure stands a minute later"); + ok(payerExposureUsd(A, now + 11 * 60_000) === 0, "an unresolved spend ages out after the stale window"); +} + +{ + __reset(); + const h = noteSpend(A, 0.25); + resolveSpend(h, false); + const snap = exposureSnapshot(); + ok(snap.length === 1 && snap[0].unsettledUsd === 0.25 && snap[0].calls === 1, + `the operator view reports who owes upstream spend (got ${JSON.stringify(snap)})`); +} + +{ + // resolveSpend must never throw on junk - it runs inside a response + // finish handler, where an exception would break the response. + __reset(); + let threw = null; + try { resolveSpend(null, true); resolveSpend({ payer: "nope", id: 9 }, true); resolveSpend(undefined, false); } + catch (e) { threw = e; } + ok(!threw, "resolving an unknown or missing handle is a no-op, never a throw"); +} + +// --- the coupling that would silently kill a tier ---------------------------- +// If the ceiling is ever smaller than the largest execution tier's underlying +// cap, that tier is dead on arrival: a single legitimate call exceeds the +// ceiling and EVERY payer is refused, honest ones included. Nothing else would +// report this - the tier would simply never succeed - so it gets an assertion +// rather than the comment it started as. +{ + __reset(); + const { EXEC_TIERS } = await import("../src/tools/route-execute.js"); + const { __config } = await import("../src/external-spend-guard.js"); + const biggest = Math.max(...EXEC_TIERS.map((t) => t.underlyingMaxUsd)); + ok(__config.DEFAULT_MAX_UNSETTLED_USD >= biggest, + `the unsettled ceiling ($${__config.DEFAULT_MAX_UNSETTLED_USD}) covers the largest tier's underlying cap ($${biggest}) - otherwise that tier can never run`); + ok(maySpend("0x1111111111111111111111111111111111111111", biggest).ok, + "a single largest-tier call is allowed for a payer with no exposure"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/test-route-execute.js b/scripts/test-route-execute.js index 9614674b..d05c415e 100644 --- a/scripts/test-route-execute.js +++ b/scripts/test-route-execute.js @@ -192,7 +192,18 @@ await expectErr({ slug: "broken-tool", params: {} }, 422, "underlying tool 422 p ok(routeExecuteHint(0.04)?.tool === "route-execute-plus", "$0.04 → plus tier boundary inclusive"); ok(routeExecuteHint(0.05)?.tool === "route-execute-max", "$0.05 → max tier (just over the plus cap)"); ok(routeExecuteHint(0.12)?.tool === "route-execute-max", "$0.12 → route-execute-max tier"); - ok(routeExecuteHint(0.9) === null, "$0.90 → no tier (above max)"); + // The pro tier (2026-08-07) exists because the $0.50 ceiling made the whole + // premium half of the index unroutable - the seller who reported the + // price:null bug prices their gates at $0.99 to $2.99, every one above the + // old max, so the router could only 409 them to their own direct route. + ok(routeExecuteHint(0.9)?.tool === "route-execute-pro", "$0.90 → route-execute-pro (was unroutable before the pro tier)"); + ok(routeExecuteHint(2.99)?.tool === "route-execute-pro", "$2.99 → pro tier - the real seller price that motivated it"); + ok(routeExecuteHint(3.0)?.tool === "route-execute-pro", "$3.00 → pro tier boundary inclusive"); + ok(routeExecuteHint(3.01) === null, "$3.01 → no tier: the ceiling still exists, it just moved"); + // The fee stays proportional rather than punishing size: 10% at the cap, the + // same spread as the max tier, not the 27x markup the plus tier was added to fix. + const pro = routeExecuteHint(3.0); + ok(Math.abs(pro.routingFeeUsd - 0.3) < 1e-9, `pro tier's fee at the cap is $0.30, a 10% spread (got ${pro.routingFeeUsd})`); } console.log(`\n${pass} passed, ${fail} failed`); diff --git a/src/external-spend-guard.js b/src/external-spend-guard.js new file mode 100644 index 00000000..7acd038c --- /dev/null +++ b/src/external-spend-guard.js @@ -0,0 +1,157 @@ +// Bound what a single buyer can make us spend upstream before they have paid us. +// +// THE HOLE. @x402/express runs the handler FIRST and settles AFTER, and a <400 +// response whose settlement then fails has its body discarded and returns 402. +// On the external routing path the handler PAYS A THIRD-PARTY SELLER from our +// spending wallet. So the sequence is: buyer's payment verifies, we pay the +// seller real USDC, our settlement fails, buyer is charged nothing. We are out +// the upstream spend with no revenue and no recourse. +// +// Verify-then-fail-to-settle is not hypothetical: it happens naturally when a +// payer's balance drops between the two (documented on Solana, where our own +// best buyer drained to $0 and its last four purchases "timed out"). Self-dealt +// it is an attack - the same wallet lists the seller and buys from it, so every +// drained dollar lands back in the attacker's pocket, bounded per call only by +// the tier cap. It scales with exactly the cap we want to raise. +// +// The existing guards bound WHAT we pay (canonical-USDC asset pin, tier cap +// re-checked against the live 402, a 50-settlement/3-payer reliability floor). +// None of them bound WHETHER WE GET PAID, because that is decided after the +// handler has already spent. +// +// So: a payer may carry only so much UNSETTLED upstream spend at once. Spend is +// recorded before the buy and resolved on the FINAL response - `res.on("finish")` +// with statusCode 200, i.e. after @x402/express has settled - which is the same +// rule the idempotency cache uses and for the same reason: a 200 is not revenue +// until settlement says so. +// +// Deliberately NOT a reputation system. It is a debt ceiling: pay for what you +// asked for and your exposure clears within seconds. It only ever bites a payer +// whose settlements are failing, which is the population it exists for. + +/** Max unsettled upstream spend one payer may carry, in USD. + * + * Sized at TWO top-tier calls (route-execute-pro covers an underlying $3.00), + * so a buyer can have a second call in flight while the first is still + * settling - agents pipeline, and refusing that would break honest traffic - + * while a wallet whose payments never settle is stopped after two rather than + * draining indefinitely. + * + * MUST be re-sized whenever a higher execution tier is added, or the new tier + * is dead on arrival: a single call larger than this ceiling is refused for + * every payer, including the honest ones. That coupling is the reason the + * tier table points back at this constant. */ +const DEFAULT_MAX_UNSETTLED_USD = Number(process.env.EXTERNAL_MAX_UNSETTLED_USD || 6.0); + +/** How long an unresolved spend counts against a payer. A settlement that never + * reports (process restart, a response that never finished) must not bar a + * payer forever, but it must not clear so fast that a fast loop outruns it. */ +const STALE_MS = Number(process.env.EXTERNAL_SPEND_STALE_MS || 10 * 60_000); + +/** Payers exempt from the ceiling: our own canary and any operator-listed + * wallet. Comma-separated, lowercased for EVM. */ +function exemptSet() { + return new Set( + String(process.env.EXTERNAL_SPEND_EXEMPT || "") + .split(",").map((s) => s.trim().toLowerCase()).filter(Boolean), + ); +} + +// payer -> [{ usd, at, settled }] +const ledger = new Map(); +let seq = 0; + +const keyOf = (payer) => { + if (typeof payer !== "string" || !payer.trim()) return null; + // EVM addresses are case-insensitive; base58/Stellar/Algorand are NOT, and + // folding them merges distinct payers (same rule as src/payer.js). + const p = payer.trim(); + return /^0x[0-9a-fA-F]{40}$/.test(p) ? p.toLowerCase() : p; +}; + +function prune(rows, now) { + return rows.filter((r) => !r.settled && now - r.at < STALE_MS); +} + +/** Current unsettled exposure for a payer, in USD. */ +export function payerExposureUsd(payer, now = Date.now()) { + const k = keyOf(payer); + if (!k) return 0; + const rows = prune(ledger.get(k) || [], now); + if (rows.length) ledger.set(k, rows); else ledger.delete(k); + return rows.reduce((s, r) => s + r.usd, 0); +} + +/** + * May this payer make us spend `usd` upstream right now? + * + * An UNKNOWN payer (free mode, a non-EIP-3009 rail we cannot attribute) is + * allowed: refusing there would break every legitimate buyer on a rail whose + * payer we cannot read, and the tier cap still bounds a single call. This is a + * per-payer debt ceiling, not an identity requirement. + */ +export function maySpend(payer, usd, { maxUnsettledUsd = DEFAULT_MAX_UNSETTLED_USD, now = Date.now() } = {}) { + const k = keyOf(payer); + if (!k) return { ok: true, reason: "payer not attributable - bounded by the tier cap alone" }; + if (exemptSet().has(k.toLowerCase())) return { ok: true, reason: "exempt" }; + const exposure = payerExposureUsd(k, now); + const next = exposure + (Number(usd) || 0); + if (next > maxUnsettledUsd) { + return { + ok: false, + exposure, + reason: + `this wallet already has $${exposure.toFixed(3)} of upstream spend from earlier calls whose payment has not settled` + + ` (ceiling $${maxUnsettledUsd}). It clears as soon as those settle.`, + }; + } + return { ok: true, exposure }; +} + +/** Record an upstream spend as UNSETTLED. Returns a handle to resolve later. */ +export function noteSpend(payer, usd, now = Date.now()) { + const k = keyOf(payer); + if (!k) return null; + const row = { id: ++seq, usd: Number(usd) || 0, at: now, settled: false }; + const rows = prune(ledger.get(k) || [], now); + rows.push(row); + ledger.set(k, rows); + return { payer: k, id: row.id }; +} + +/** + * Resolve a recorded spend once the FINAL response is known. + * + * `settled` must come from the post-settlement status (res.on("finish") with + * statusCode === 200), never from the handler's own return: the handler + * succeeding is precisely the state that precedes a settlement failure. + */ +export function resolveSpend(handle, settled) { + if (!handle?.payer) return; + const rows = ledger.get(handle.payer); + if (!rows) return; + const row = rows.find((r) => r.id === handle.id); + if (!row) return; + if (settled) { + row.settled = true; + const left = rows.filter((r) => !r.settled); + if (left.length) ledger.set(handle.payer, left); else ledger.delete(handle.payer); + } + // NOT settled: the row stays as exposure until it ages out. That is the + // whole point - an unpaid spend must keep counting against the payer. +} + +/** Operator view: who currently owes us upstream spend. Counts only. */ +export function exposureSnapshot(now = Date.now()) { + const out = []; + for (const [payer, rows] of ledger) { + const live = prune(rows, now); + if (!live.length) continue; + out.push({ payer, unsettledUsd: Number(live.reduce((s, r) => s + r.usd, 0).toFixed(6)), calls: live.length }); + } + return out.sort((a, b) => b.unsettledUsd - a.unsettledUsd); +} + +/** Test-only. */ +export function __reset() { ledger.clear(); seq = 0; } +export const __config = { DEFAULT_MAX_UNSETTLED_USD, STALE_MS }; diff --git a/src/payments.js b/src/payments.js index 1353fa0f..595cb011 100644 --- a/src/payments.js +++ b/src/payments.js @@ -320,11 +320,11 @@ export const isIdentityBoundRoute = (def) => // Router tiers only - the subset whose Algorand revenue is chain-matched to // the AVM spending wallet (see avmPayToFor). Blockscout stays out: its // upstream spend is Base-pinned regardless of the buyer's rail. -export const AVM_SELF_FUNDING_SLUGS = new Set(["route-execute", "route-execute-plus", "route-execute-max"]); +export const AVM_SELF_FUNDING_SLUGS = new Set(["route-execute", "route-execute-plus", "route-execute-max", "route-execute-pro"]); const AVM_UPSTREAM_BUYER_ADDRESS = (process.env.ALGORAND_UPSTREAM_BUYER_ADDRESS || "").trim(); export const SELF_FUNDING_SLUGS = new Set([ - "route-execute", "route-execute-plus", "route-execute-max", + "route-execute", "route-execute-plus", "route-execute-max", "route-execute-pro", // Blockscout kit (2026-07-29, the house rule: everything that spends from the // burner settles to the burner): each call pays Blockscout ~$0.002 upstream // from the same wallet, so treasury-settled revenue was a slow one-way diff --git a/src/server.js b/src/server.js index 75a468fa..5e004096 100644 --- a/src/server.js +++ b/src/server.js @@ -24,6 +24,7 @@ import { PERSISTENT as memoryPersistent, } from "./tools/memory.js"; import { payerFromRequest, payerFromPaymentResponse } from "./payer.js"; +import { resolveSpend as resolveExternalSpend, exposureSnapshot } from "./external-spend-guard.js"; import { registerWellKnown, removeWellKnown, getWellKnown, listWellKnown } from "./well-known-store.js"; import { backupPlan, backupStatus, runBackup, startBackupScheduler } from "./backup.js"; import { assertAvmValidityCovers } from "./avm-validity.js"; @@ -4474,6 +4475,21 @@ for (const tool of ALL_KIT) { const result = await tool.handler(input, req); + // A handler that spent real money upstream (external route-execute) leaves + // a handle on the request. Resolve it against the FINAL response, not the + // handler's return: settlement runs after this, and handler-success is + // precisely the state that precedes a settlement failure. Only a settled + // 200 clears the payer's exposure; anything else leaves it standing, which + // is what stops a wallet whose payments never settle from draining the + // upstream wallet one call at a time. Same doctrine as the idempotency + // cache's commit-on-finish. + if (req.__externalSpend) { + const handle = req.__externalSpend; + res.on("finish", () => { + try { resolveExternalSpend(handle, res.statusCode === 200); } catch { /* never break a response */ } + }); + } + if (cachePolicy) { noteCacheOutcome(cacheKey ? "miss" : "skip"); res.setHeader("X-Cache", cacheKey ? "miss" : "skip"); diff --git a/src/tools/route-execute.js b/src/tools/route-execute.js index 06d33bc9..a80ecd4a 100644 --- a/src/tools/route-execute.js +++ b/src/tools/route-execute.js @@ -17,7 +17,8 @@ // that names the tool and its direct route, so the buyer can call it at list // price instead. import { createHash } from "node:crypto"; -import { paymentHeaderOf } from "../payer.js"; +import { paymentHeaderOf, payerFromRequest } from "../payer.js"; +import { maySpend, noteSpend, resolveSpend } from "../external-spend-guard.js"; import { findTools } from "../find.js"; import { isIdentityBoundRoute } from "../payments.js"; @@ -41,6 +42,18 @@ export const EXEC_TIERS = [ // external inventory the seller review identified as routable demand. { slug: "route-execute-plus", execPriceUsd: 0.05, underlyingMaxUsd: 0.04 }, { slug: "route-execute-max", execPriceUsd: 0.55, underlyingMaxUsd: 0.5 }, + // 2026-08-07: the $0.50 ceiling made the whole premium half of the index + // unroutable. The seller who reported the price:null bug prices their gates + // at $0.99, $1.50 and $2.99 - every one of them above the top tier, so the + // router could only 409 them at their own direct route. Same 10% spread as + // the max tier, so the curve stays proportional rather than punishing size. + // + // This tier is only safe because of the per-payer debt ceiling in + // external-spend-guard.js. Settlement runs AFTER the handler, so raising the + // cap raises exactly one exposure: what a buyer whose payment verifies and + // then fails to settle can make us spend before we stop them. Keep + // EXTERNAL_MAX_UNSETTLED_USD sized against THIS number, not the old $0.50. + { slug: "route-execute-pro", execPriceUsd: 3.3, underlyingMaxUsd: 3.0 }, ]; const EXEC_SLUGS = new Set(EXEC_TIERS.map((t) => t.slug)); /** Which execution tier (if any) can run a tool at `underlyingUsd`, and the @@ -258,10 +271,33 @@ export function buildRouteExecuteTool({ getCatalog, baseUrl = "", tier = EXEC_TI extUrl = qs ? `${ext.url}${ext.url.includes("?") ? "&" : "?"}${qs}` : ext.url; extBody = undefined; } + // PER-PAYER DEBT CEILING. Everything above bounds WHAT we pay (the + // canonical-USDC asset pin, this tier's cap re-checked against the + // live 402, the 50-settlement reliability floor). Nothing bounds + // WHETHER WE GET PAID: settlement runs AFTER this handler, so a + // payment that verifies and then fails to settle leaves us having + // spent real USDC upstream for a buyer who is charged nothing. + // Self-dealt - one wallet listing the seller and buying from it - + // every drained dollar returns to the attacker, bounded per call only + // by `cap`. A 4xx here cancels the buyer's settlement, so refusing + // costs an honest buyer nothing. + const spendPayer = payerFromRequest(req); + const allowed = maySpend(spendPayer, extUsd); + if (!allowed.ok) throw bad(`External routing is paused for this wallet: ${allowed.reason}`, 429); + const spendHandle = noteSpend(spendPayer, extUsd); + // Handed to server.js on the REQUEST, because a tool handler is called + // as handler(input, req) and never receives `res`. The first draft + // registered res.on("finish") here, where `res` is undefined - a guard + // that installs nothing and reports no error, which is the exact shape + // of defect this session keeps finding. server.js resolves it on the + // FINAL response (post-settlement), never on handler success. + if (spendHandle && req && typeof req === "object") req.__externalSpend = spendHandle; let paid; try { paid = await payExternal(extUrl, { method: extMethod, body: extBody, maxAtomic: BigInt(Math.round(cap * 1e6)), chain }); } catch (e) { + // We never spent, so it is not exposure. + try { resolveSpend(spendHandle, true); } catch { /* best effort */ } const sc = e?.statusCode && e.statusCode >= 400 && e.statusCode < 600 ? e.statusCode : 502; throw bad(`External seller "${ext.seller}" failed: ${String(e?.message || e).slice(0, 200)}`, sc); } From a22f4f477c818b20a7c073404aac97882ea982c7 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:30:24 -0400 Subject: [PATCH 05/10] [test][deploy] Trigger CI for the spend guard and pro tier Co-Authored-By: Claude Opus 5 (1M context) --- .github/trigger-deploy | 2 +- .github/trigger-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 13c6292f..09b2d554 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -1 +1 @@ -1786115711 +1786116624 diff --git a/.github/trigger-test b/.github/trigger-test index 13c6292f..09b2d554 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786115711 +1786116624 From 4eaa757af7d345d1b0445b107bb2f6bfb2b8f9ac Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:32:52 -0400 Subject: [PATCH 06/10] [test][deploy] Size the spending wallet's alarm against the biggest call, not the smallest Raising the routing ceiling to $3.00 quietly broke the alarm that watches the wallet paying for it. UPSTREAM_BUYER_LOW_USD defaulted to $0.50, which was correct when the only thing spending from that wallet was Blockscout at $0.002/call - $0.50 covered hundreds of calls. With route-execute-pro, "ok" means "has at least $0.50" for a wallet that cannot fund ONE call, so /api/gateway-status would have reported ok and the heartbeat would have stayed green right up to the failure the alarm exists to prevent. Measured before writing this: prod reports upstreamBuyer "ok" right now, and "ok" is a bucket, not a balance - it cannot distinguish $0.60 from $60. Default is now two largest-tier calls, so we are paged with room to top up rather than at the moment of starvation, and an assertion locks it to EXEC_TIERS: a threshold that silently stops covering the biggest call reports nothing on its own. Same coupling, and same fix, as the unsettled-spend ceiling in the previous commit - adding a tier now fails CI in two places instead of degrading two guards in silence. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/test-route-execute.js | 14 ++++++++++++++ src/tools/blockscout-kit.js | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/test-route-execute.js b/scripts/test-route-execute.js index d05c415e..ca238872 100644 --- a/scripts/test-route-execute.js +++ b/scripts/test-route-execute.js @@ -206,5 +206,19 @@ await expectErr({ slug: "broken-tool", params: {} }, 422, "underlying tool 422 p ok(Math.abs(pro.routingFeeUsd - 0.3) < 1e-9, `pro tier's fee at the cap is $0.30, a 10% spread (got ${pro.routingFeeUsd})`); } + +// --- the spending wallet's alarm must cover the biggest call ------------------ +// $0.50 was right when only Blockscout ($0.002/call) spent from that wallet. +// A tier that can spend $3.00 in one call makes "ok" mean "has at least $0.50" +// for a wallet that cannot cover a single call - the alarm would stay green +// right up to the failure it exists to prevent. Nothing else reports this. +{ + const { BUYER_LOW_DEFAULT_USD } = await import("../src/tools/blockscout-kit.js"); + const { EXEC_TIERS: TIERS } = await import("../src/tools/route-execute.js"); + const biggest = Math.max(...TIERS.map((t) => t.underlyingMaxUsd)); + ok(BUYER_LOW_DEFAULT_USD >= biggest, + `the upstream buyer low-water default ($${BUYER_LOW_DEFAULT_USD}) covers the largest tier's underlying spend ($${biggest}) - otherwise "ok" can mean "cannot fund one call"`); +} + console.log(`\n${pass} passed, ${fail} failed`); process.exit(fail ? 1 : 0); diff --git a/src/tools/blockscout-kit.js b/src/tools/blockscout-kit.js index decc2033..6ed2d0e9 100644 --- a/src/tools/blockscout-kit.js +++ b/src/tools/blockscout-kit.js @@ -290,7 +290,22 @@ export const BLOCKSCOUT_TOOLS = [ // with graceful "unknown" (an RPC flake must never page). const BASE_RPCS = ["https://mainnet.base.org", "https://base.llamarpc.com", "https://base.drpc.org"]; const USDC_BASE = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"; -const BUYER_LOW_USD = () => Number(process.env.UPSTREAM_BUYER_LOW_USD || "0.5"); +// Sized against the LARGEST single spend this wallet can be asked to make, not +// against the smallest. +// +// $0.50 was right when the only thing spending from here was Blockscout at +// $0.002/call: a wallet above it could serve hundreds of calls. Then +// route-execute-pro (2026-08-07) made a single call able to spend $3.00 +// upstream, and "ok" started meaning "has at least $0.50" for a wallet that +// could not cover one call. The alarm would have stayed green right up to the +// failure it exists to prevent. +// +// Two largest-tier calls, so we are paged with room to top up rather than at +// the moment of starvation. MUST be re-sized whenever a bigger execution tier +// lands - locked by an assertion in scripts/test-route-execute.js, because a +// threshold that quietly stops covering the biggest call reports nothing. +export const BUYER_LOW_DEFAULT_USD = 6; +const BUYER_LOW_USD = () => Number(process.env.UPSTREAM_BUYER_LOW_USD || String(BUYER_LOW_DEFAULT_USD)); const BUYER_STATUS_CACHE_MS = 5 * 60_000; let buyerStatusCache = null; /** Bucketed BALANCE of the Base spending wallet. Nothing more. From d6ba5a8d628f167d4c2e457f6cb966d5970921a7 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:32:52 -0400 Subject: [PATCH 07/10] [test][deploy] Trigger CI for the buyer low-water resize Co-Authored-By: Claude Opus 5 (1M context) --- .github/trigger-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/trigger-test b/.github/trigger-test index 09b2d554..c98e3237 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786116624 +1786116772 From 2b71744dbdeff81df899dd25308089f564036574 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:35:17 -0400 Subject: [PATCH 08/10] [test][deploy] A self-funding wallet that FALLS is the alarm, not one that runs low Mike's observation, and it holds up in the code: that wallet should never go down. Everything that spends from it also settles INTO it - SELF_FUNDING_SLUGS sets payTo to the burner for exactly those tools (payments.js acceptsForItem) - and every execution tier charges more than it can spend. Worst case per call: +$0.005, +$0.01, +$0.05, +$0.30. Blockscout is the same shape. So barring a manual withdrawal the balance is monotonically non-decreasing. Which makes the low-water threshold the wrong instrument. It answers "is there enough left", and only after the money has gone. The interesting question is "did it fall at all", because a fall means one of exactly three things: someone withdrew, a spend's revenue never arrived (the verify-then-fail-to-settle drain the per-payer ceiling now bounds), or something we do not understand. A withdrawal trips it too, on purpose. The alarm's job is to say "this wallet fell and nobody told me"; a human who withdrew closes the issue in one click, and a silent fall is the one we must never miss. The hard part is the dip that settlement ordering GUARANTEES: we pay the seller during the handler and collect afterwards, so the balance is legitimately lower in between. An alarm that cannot tell that from a drain would page on every healthy call and be muted within a day. Hence a high-water mark, a tolerance, and a requirement that the fall persist across consecutive 5-minute reads. One subtlety with its own assertion: a read INSIDE tolerance must not reset the counter. A wallet bleeding $0.40 per read sits within tolerance every single time, so a counter that cleared on those would never fire while the wallet quietly emptied. Balances stay off /api/gateway-status - the trend is bucketed to "ok" / "draining" / "unknown", never a number, same rule as the existing status. 16 offline assertions, including the false positive that matters (an in-flight dip that recovers) and the recovery path that clears the issue. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/deploy.yml | 3 ++ .github/workflows/heartbeat.yml | 34 +++++++++++++ scripts/test-buyer-balance-trend.js | 78 +++++++++++++++++++++++++++++ src/tools/blockscout-kit.js | 46 ++++++++++++++++- 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 scripts/test-buyer-balance-trend.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 779d5180..007eed70 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1015,6 +1015,9 @@ jobs: - name: "Live 402 quote (a priceless row learns its price from the challenge; unpriceable never becomes $0; PUT/PATCH/DELETE never probed — offline)" run: node scripts/test-x402-live-quote.js + - name: "Buyer wallet trend (a self-funding wallet that FALLS is unexplained; a transient in-flight dip is not — offline)" + run: node scripts/test-buyer-balance-trend.js + - name: "External spend guard (per-payer debt ceiling: an UNSETTLED upstream spend keeps counting, so verify-then-fail-to-settle cannot drain the wallet — offline)" run: node scripts/test-external-spend-guard.js diff --git a/.github/workflows/heartbeat.yml b/.github/workflows/heartbeat.yml index 58833c62..6054510c 100644 --- a/.github/workflows/heartbeat.yml +++ b/.github/workflows/heartbeat.yml @@ -260,6 +260,40 @@ jobs: gh issue close "$OPEN" --repo "$GITHUB_REPOSITORY" --comment "Recovered: balance back above the low-water mark at $(date -u +%FT%TZ)." fi + # The spending wallet should never go DOWN, so a fall is worth more than + # a floor. Everything that spends from it also settles into it + # (SELF_FUNDING_SLUGS), and every execution tier charges more than it can + # spend, so barring a manual withdrawal the balance only rises. A + # low-water alarm fires after the money is gone; this fires on the first + # unexplained dollar. A withdrawal trips it too, deliberately: the alarm's + # job is to say "this wallet fell and nobody told me", and a human who + # withdrew can close the issue in one click. + - name: Upstream buyer wallet trend (a self-funding wallet that FALLS is unexplained) + if: always() + continue-on-error: true + run: | + TREND=$(curl -s --max-time 15 "$PROD/api/gateway-status" | jq -r '.upstreamBuyer.trend // "unknown"' 2>/dev/null || echo unknown) + echo "upstream buyer trend: $TREND" + TITLE="Upstream buyer wallet is DRAINING (unexplained fall)" + OPEN=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ "$TREND" = "draining" ]; then + if [ -z "$OPEN" ]; then + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body "The x402 upstream spending wallet has fallen below its high-water mark across several consecutive reads. + + That wallet is SELF-FUNDING: every tool that spends from it also settles into it, and every execution tier charges more than it can spend (worst case +\$0.005 / +\$0.01 / +\$0.05 / +\$0.30 per call). Its balance should only rise. A sustained fall means one of: + + 1. A manual withdrawal - close this issue if that was you. + 2. Upstream spend whose revenue never arrived: a buyer's payment verified and then failed to settle, which is the drain the per-payer ceiling in src/external-spend-guard.js bounds. Check /__operator/stats and the route-execute receipts. + 3. Something we do not understand, which is why this alarm exists. + + Balances are deliberately not published on /api/gateway-status; read the wallet directly to see the number." + else + gh issue comment "$OPEN" --repo "$GITHUB_REPOSITORY" --body "Still falling at $(date -u +%FT%TZ)." + fi + elif [ "$TREND" = "ok" ] && [ -n "$OPEN" ]; then + gh issue close "$OPEN" --repo "$GITHUB_REPOSITORY" --comment "Recovered: the wallet set a new high-water mark at $(date -u +%FT%TZ)." + fi + # Settlement freshness — the alarm that did not exist on 2026-08-07. # # The daily paid canary stopped buying on 2026-08-02 and reported success diff --git a/scripts/test-buyer-balance-trend.js b/scripts/test-buyer-balance-trend.js new file mode 100644 index 00000000..11bb65cc --- /dev/null +++ b/scripts/test-buyer-balance-trend.js @@ -0,0 +1,78 @@ +// The spending wallet should never go down, so a FALL is the signal. +// +// Everything that spends from it settles into it (SELF_FUNDING_SLUGS), and every +// execution tier charges more than it can spend. Barring a manual withdrawal the +// balance is monotonically non-decreasing. A low-water alarm fires after the +// money is gone; this fires on the first unexplained dollar. +// +// The hard part is the TRANSIENT dip that settlement ordering guarantees: we pay +// the seller during the handler and collect afterwards. An alarm that cannot +// tell that from a drain would page on every healthy call. +import { noteBuyerBalance } from "../src/tools/blockscout-kit.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; +const fresh = (v) => noteBuyerBalance(v, { reset: true }); + +{ + ok(fresh(10) === "ok", "the first read is a baseline, never an alarm"); + ok(noteBuyerBalance(10.3) === "ok", "a rise is healthy - this is what a self-funding wallet does"); + ok(noteBuyerBalance(10.6) === "ok", "…and keeps re-baselining upward"); +} + +{ + // THE FALSE POSITIVE THIS MUST NOT HAVE. Settlement runs after the handler, + // so the balance dips while a call is in flight and recovers when revenue + // lands. Paging on that would make the alarm useless within a day. + fresh(10); + ok(noteBuyerBalance(9.7) === "ok", "a dip inside tolerance is an in-flight call, not a drain"); + ok(noteBuyerBalance(10.4) === "ok", "and it recovers when the buyer's payment settles"); +} + +{ + // THE REAL DRAIN. A fall past tolerance, sustained across consecutive reads. + fresh(10); + ok(noteBuyerBalance(7) === "ok", "one big fall is not yet an alarm - a single read could be an in-flight max-tier call"); + ok(noteBuyerBalance(6.9) === "ok", "two is still not"); + ok(noteBuyerBalance(6.8) === "draining", "three consecutive reads below the high-water mark is a drain"); +} + +{ + // A slow bleed sits INSIDE tolerance on every individual read. If a + // within-tolerance read cleared the counter, a wallet losing $0.40 a read + // would never alarm - it would just quietly empty. + fresh(10); + noteBuyerBalance(6.5); // past tolerance, counter 1 + noteBuyerBalance(6.4); // counter 2 + ok(noteBuyerBalance(9.6) === "ok", + "a read within tolerance of the high-water mark does not itself alarm"); + fresh(10); + ok([9.6, 9.55, 9.5].map((v) => noteBuyerBalance(v)).every((s) => s === "ok"), + "…and a genuinely small wobble never alarms on its own"); +} + +{ + // Recovery must clear it, or one bad afternoon pages forever. + fresh(10); + noteBuyerBalance(6); noteBuyerBalance(5.9); + ok(noteBuyerBalance(5.8) === "draining", "draining while it is falling"); + ok(noteBuyerBalance(11) === "ok", "a new high clears the alarm and re-baselines"); + ok(noteBuyerBalance(10.9) === "ok", "and the counter really was reset, not merely masked"); +} + +{ + ok(fresh(NaN) === "unknown", "an unreadable balance is unknown, never a drain"); + ok(noteBuyerBalance(undefined) === "unknown", "and never throws on junk"); +} + +{ + // A withdrawal looks exactly like a drain, and SHOULD: the alarm's job is to + // say "this wallet fell and nobody told me". A human who withdrew can close + // the issue; a silent fall is the one we must never miss. + fresh(50); + const seq = [20, 20, 20].map((v) => noteBuyerBalance(v)); + ok(seq[2] === "draining", "a manual withdrawal alarms too - indistinguishable on purpose"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/src/tools/blockscout-kit.js b/src/tools/blockscout-kit.js index 6ed2d0e9..1b6beba0 100644 --- a/src/tools/blockscout-kit.js +++ b/src/tools/blockscout-kit.js @@ -305,6 +305,50 @@ const USDC_BASE = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"; // lands - locked by an assertion in scripts/test-route-execute.js, because a // threshold that quietly stops covering the biggest call reports nothing. export const BUYER_LOW_DEFAULT_USD = 6; + +// THIS WALLET SHOULD NEVER GO DOWN, so a fall is worth more than a floor. +// +// Everything that spends from it also settles INTO it: SELF_FUNDING_SLUGS sets +// payTo to this address for exactly those tools (payments.js acceptsForItem), +// and every execution tier charges more than it can spend - worst case +$0.005, +// +$0.01, +$0.05, +$0.30 per call. Blockscout is the same shape ($0.002 upstream +// against a priced tool). So barring a manual withdrawal the balance is +// monotonically non-decreasing, and a SUSTAINED fall means something we do not +// understand is happening: the verify-then-fail-to-settle drain, a spend whose +// revenue never arrived, or a withdrawal nobody mentioned. +// +// A low-water alarm fires after the money is gone. This fires on the first +// unexplained dollar, which is the whole difference. +// +// It must tolerate a TRANSIENT dip, because settlement ordering guarantees one: +// we pay the seller during the handler and collect afterwards, so the balance +// is legitimately lower in between. Hence a high-water mark, a tolerance, and a +// requirement that the fall persist across consecutive reads (each 5 min apart) +// before it is called draining. +const BUYER_DROP_TOLERANCE_USD = Number(process.env.UPSTREAM_BUYER_DROP_TOLERANCE_USD || 0.5); +const BUYER_DROP_READS = Number(process.env.UPSTREAM_BUYER_DROP_READS || 3); +let buyerHighWater = null; +let buyerBelowReads = 0; + +/** Bucketed trend for the spending wallet: "ok" | "draining". Never a number - + * /api/gateway-status is public and balances stay off it. Exported for tests. */ +export function noteBuyerBalance(balance, { reset = false } = {}) { + if (reset) { buyerHighWater = null; buyerBelowReads = 0; } + if (!Number.isFinite(balance)) return "unknown"; + if (buyerHighWater == null || balance >= buyerHighWater) { + // A new high (or the first read) is the healthy case: re-baseline and clear. + buyerHighWater = balance; + buyerBelowReads = 0; + return "ok"; + } + if (buyerHighWater - balance <= BUYER_DROP_TOLERANCE_USD) { + // Within tolerance: an in-flight call, not a drain. Do NOT reset the + // counter - a slow bleed sits inside tolerance on every single read. + return buyerBelowReads >= BUYER_DROP_READS ? "draining" : "ok"; + } + buyerBelowReads += 1; + return buyerBelowReads >= BUYER_DROP_READS ? "draining" : "ok"; +} const BUYER_LOW_USD = () => Number(process.env.UPSTREAM_BUYER_LOW_USD || String(BUYER_LOW_DEFAULT_USD)); const BUYER_STATUS_CACHE_MS = 5 * 60_000; let buyerStatusCache = null; @@ -349,7 +393,7 @@ export async function upstreamBuyerStatus() { } result = balance == null ? { configured: true, status: "unknown", attests: "balance-only" } - : { configured: true, status: balance < BUYER_LOW_USD() ? "low" : "ok", attests: "balance-only" }; + : { configured: true, status: balance < BUYER_LOW_USD() ? "low" : "ok", attests: "balance-only", trend: noteBuyerBalance(balance) }; } catch { result = { configured: true, status: "unknown", attests: "balance-only" }; } From 3e1062173790b665593288e1729dc7885a4b7e26 Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:35:17 -0400 Subject: [PATCH 09/10] [test][deploy] Trigger CI for the buyer wallet trend alarm Co-Authored-By: Claude Opus 5 (1M context) --- .github/trigger-deploy | 2 +- .github/trigger-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 09b2d554..7ff28fd3 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -1 +1 @@ -1786116624 +1786116917 diff --git a/.github/trigger-test b/.github/trigger-test index c98e3237..7ff28fd3 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786116772 +1786116917 From 740eba9f58fdb256a10329376aaac419a15c959e Mon Sep 17 00:00:00 2001 From: mikeypetrillo <22065635+MikeyPetrillo@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:44:20 -0400 Subject: [PATCH 10/10] [test][deploy] Self-review of today's new code: three defects, a redaction, and the redaction's own leak Reviewed everything I added today instead of assuming it was clean. GLOBAL PROBE BUDGET (the one that mattered). The live-quote enrichment was bounded per SELLER at three routes per crawl, which sounds gentle until you multiply: roughly a third of indexed rows carry no price, so a per-seller-only limit fires thousands of outbound requests every five minutes across the whole index. That is issue #645 rebuilt with a different label - the report about 686 requests to a path that 404'd every time. Per-route backoff quiets it eventually, but "eventually" is the first several cycles, and the seller feels those. Now bounded per CYCLE as well; the rest wait their turn. EXPOSURE CLEARED ON A THROW. The external-spend catch cleared the payer's exposure on the theory that a failed buy never spent. payExternal can throw AFTER signing and broadcasting - a network error on the response, a timeout - and clearing on those is precisely how a spend disappears from the ledger the guard exists to keep. The exposure now stands and ages out on its own. DEAD IMPORT. exposureSnapshot was imported into server.js and never used. It returns payer addresses, so the load-bearing fact is that it was never wired to a public surface; the import is gone. REDACTION, AND ITS OWN LEAK. The facilitator diagnostic logs a body we did not write, from a host we authenticate to, into a log aggregator - so long hex/base64 runs are redacted, since an error page echoing a request header would put a credential there permanently. The first version walked straight past a JWT (segments are dot-separated and individually short), so a bearer token would have been logged whole. Then the fixture I wrote to prove that failed our own gitleaks history scan: RuleID jwt. A realistic three-segment token in the repo is exactly what that rule is for, and .gitleaks.toml is explicit that paths are never excluded because a path exclude would hide a real credential in a test fixture. So the fixture is now shape-only with no eyJ prefix - it exercises the same branch without putting a token-shaped string in the repo - and the commit that carried it was rewritten out of the branch rather than allowlisted. Also asserted: a Cloudflare ray id must SURVIVE redaction, since it is the thing a provider asks for. Checked and found fine: price stays polymorphic (number from bazaar, "$x" from manifests) but every surface reads it through parsePrice/fmtUsd. Local sweep green: facilitator-diagnostics 35, live-quote 28, spend-guard 20, balance-trend 16, route-execute 46, wish 54, canary-coverage 148, and the MCP self-consistency guard at 1467 against a booted server. Co-Authored-By: Claude Opus 5 (1M context) --- .github/trigger-deploy | 2 +- .github/trigger-test | 2 +- scripts/test-facilitator-diagnostics.js | 25 +++++++++++++++++++++++++ src/facilitator-diagnostics.js | 14 +++++++++++++- src/server.js | 2 +- src/tools/route-execute.js | 9 +++++++-- src/x402-index.js | 13 ++++++++++++- 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 7ff28fd3..6ebecaee 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -1 +1 @@ -1786116917 +1786117290 diff --git a/.github/trigger-test b/.github/trigger-test index 7ff28fd3..6ebecaee 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786116917 +1786117290 diff --git a/scripts/test-facilitator-diagnostics.js b/scripts/test-facilitator-diagnostics.js index c01b061f..d91b4050 100644 --- a/scripts/test-facilitator-diagnostics.js +++ b/scripts/test-facilitator-diagnostics.js @@ -187,5 +187,30 @@ const CF_BLOCK = ` Coinbase ${secretish}` }); + ok(!line.includes("abababab"), "a long hex run is redacted out of the logged excerpt"); + ok(!line.includes("pppppppppppp"), "a three-segment bearer-token shape is redacted too - the shape a plain long-run rule walks past"); + ok(line.includes("[redacted]"), "…and the redaction is visible rather than silent"); + ok(/Error for key/.test(line), "the surrounding words survive - the diagnosis is still readable"); + // A Cloudflare ray id is short and must NOT be swallowed by the redaction. + const ray = describeErrorResponse({ url: "https://f.example/settle", status: 502, headers: { "cf-ray": "8f2c1d4e5a6b7c8d-ATL" }, body: "Access denied" }); + ok(ray.includes("8f2c1d4e5a6b7c8d-ATL"), "a ray id survives redaction - it is the thing a provider asks for"); +} + console.log(`\n${pass} passed, ${fail} failed`); process.exit(fail ? 1 : 0); diff --git a/src/facilitator-diagnostics.js b/src/facilitator-diagnostics.js index c7197621..5500f38c 100644 --- a/src/facilitator-diagnostics.js +++ b/src/facilitator-diagnostics.js @@ -91,7 +91,19 @@ export function describeErrorResponse({ url, status, headers, body } = {}) { .filter(Boolean); // 600 chars of WORDS, having spent none of the budget on markup. The page // that started this fits its whole message inside it. - const excerpt = text.length > 600 ? `${text.slice(0, 597)}...` : text; + // + // Long hex/base64 runs are REDACTED first. We are logging a body we did not + // write, from a host we authenticate to, straight into a log aggregator - and + // an error page that echoes a request header or a signed payload would put a + // credential there permanently. Nothing diagnostic is lost: a Cloudflare ray + // id is short, and no failure has ever been explained by a 64-character blob. + const redacted = text + // JWT-shaped first: its segments are dot-separated and individually short, + // so a plain "long run" rule walks straight past a whole bearer token. + .replace(/\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[redacted]") + .replace(/\b(?:0x)?[0-9a-fA-F]{32,}\b/g, "[redacted]") + .replace(/\b[A-Za-z0-9+/_-]{40,}={0,2}\b/g, "[redacted]"); + const excerpt = redacted.length > 600 ? `${redacted.slice(0, 597)}...` : redacted; return `[facilitator-diag] ${status} from ${url} - ${verdict}` + `${bits.length ? ` | ${bits.join(" ")}` : ""}` + `${excerpt ? ` | body: ${excerpt}` : " | body: "}`; diff --git a/src/server.js b/src/server.js index 5e004096..6c572104 100644 --- a/src/server.js +++ b/src/server.js @@ -24,7 +24,7 @@ import { PERSISTENT as memoryPersistent, } from "./tools/memory.js"; import { payerFromRequest, payerFromPaymentResponse } from "./payer.js"; -import { resolveSpend as resolveExternalSpend, exposureSnapshot } from "./external-spend-guard.js"; +import { resolveSpend as resolveExternalSpend } from "./external-spend-guard.js"; import { registerWellKnown, removeWellKnown, getWellKnown, listWellKnown } from "./well-known-store.js"; import { backupPlan, backupStatus, runBackup, startBackupScheduler } from "./backup.js"; import { assertAvmValidityCovers } from "./avm-validity.js"; diff --git a/src/tools/route-execute.js b/src/tools/route-execute.js index a80ecd4a..c7f55497 100644 --- a/src/tools/route-execute.js +++ b/src/tools/route-execute.js @@ -296,8 +296,13 @@ export function buildRouteExecuteTool({ getCatalog, baseUrl = "", tier = EXEC_TI try { paid = await payExternal(extUrl, { method: extMethod, body: extBody, maxAtomic: BigInt(Math.round(cap * 1e6)), chain }); } catch (e) { - // We never spent, so it is not exposure. - try { resolveSpend(spendHandle, true); } catch { /* best effort */ } + // The exposure DELIBERATELY stands. It is tempting to clear it here + // ("the buy failed, so we never spent"), but payExternal can throw + // after signing and broadcasting - a network error on the response, + // a timeout - and clearing on those is exactly the case that lets a + // spend disappear from the ledger. It ages out on its own within + // the stale window, so an honest buyer caught by a seller outage + // waits, while a spend we cannot account for keeps counting. const sc = e?.statusCode && e.statusCode >= 400 && e.statusCode < 600 ? e.statusCode : 502; throw bad(`External seller "${ext.seller}" failed: ${String(e?.message || e).slice(0, 200)}`, sc); } diff --git a/src/x402-index.js b/src/x402-index.js index baeed937..29ecd0fc 100644 --- a/src/x402-index.js +++ b/src/x402-index.js @@ -1152,6 +1152,15 @@ function paywallProbeDue() { // price); one that cannot be priced backs off through probeDue like every // other path. See the #645 note below on why per-PATH backoff matters. const LIVE_QUOTE_PROBES_PER_CRAWL = 3; +// GLOBAL ceiling per crawl CYCLE, not just per seller. Three per seller sounds +// gentle until you multiply: roughly a third of indexed rows carry no price, so +// a per-seller-only limit fires thousands of outbound requests every 5 minutes +// across the whole index - which is issue #645 rebuilt with a different label. +// Per-route backoff eventually quiets the sellers who never answer 402, but +// "eventually" is the first several cycles, and the seller feels those. This +// bounds the whole cycle; the rest simply wait their turn on the next one. +const LIVE_QUOTE_PROBES_PER_CYCLE = Number(process.env.LIVE_QUOTE_PROBES_PER_CYCLE || 60); +let liveQuoteBudget = LIVE_QUOTE_PROBES_PER_CYCLE; /** * Learn price + networks from a live 402 for rows that have neither. @@ -1177,8 +1186,9 @@ async function enrichLiveQuotes(tools, originUrl) { && !(Array.isArray(t.networks) && t.networks.length) // already payable-evidenced && probeMethodsFor(t).length // never PUT/PATCH/DELETE && probeDue(originUrl, `quote:${t.route}`), - ).slice(0, LIVE_QUOTE_PROBES_PER_CRAWL); + ).slice(0, Math.max(0, Math.min(LIVE_QUOTE_PROBES_PER_CRAWL, liveQuoteBudget))); if (!candidates.length) return tools; + liveQuoteBudget -= candidates.length; const { assertPublicUrl, ssrfDispatcher } = await import("./tools/fetch-guard.js"); for (const tool of candidates) { @@ -1690,6 +1700,7 @@ async function runCrawl() { crawlInFlight = true; try { const seeds = seedList(); + liveQuoteBudget = LIVE_QUOTE_PROBES_PER_CYCLE; // fresh allowance each cycle await runPool(seeds, CRAWL_CONCURRENCY, crawlSeller); } finally { crawlInFlight = false;