diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 07da5bc0..6ebecaee 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -1 +1 @@ -1786112186 +1786117290 diff --git a/.github/trigger-test b/.github/trigger-test index aee45a80..6ebecaee 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -1 +1 @@ -1786111133 +1786117290 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 021016f1..007eed70 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1012,6 +1012,15 @@ jobs: - name: Boot /supported guard (a dead facilitator costs ONE rail, not every paid route — the 2026-08-01 Celo outage; probe-driven drop, fail-open on total blindness, escape hatch — offline) run: node scripts/test-supported-guard.js + - name: "Live 402 quote (a priceless row learns its price from the challenge; unpriceable never becomes $0; PUT/PATCH/DELETE never probed — offline)" + run: node scripts/test-x402-live-quote.js + + - name: "Buyer wallet trend (a self-funding wallet that FALLS is unexplained; a transient in-flight dip is not — offline)" + run: node scripts/test-buyer-balance-trend.js + + - name: "External spend guard (per-payer debt ceiling: an UNSETTLED upstream spend keeps counting, so verify-then-fail-to-settle cannot drain the wallet — offline)" + run: node scripts/test-external-spend-guard.js + - name: "Facilitator diagnostics (an HTML error page must name itself: block vs rate limit vs origin failure, and never touch the response — offline)" run: node scripts/test-facilitator-diagnostics.js diff --git a/.github/workflows/heartbeat.yml b/.github/workflows/heartbeat.yml index 58833c62..6054510c 100644 --- a/.github/workflows/heartbeat.yml +++ b/.github/workflows/heartbeat.yml @@ -260,6 +260,40 @@ jobs: gh issue close "$OPEN" --repo "$GITHUB_REPOSITORY" --comment "Recovered: balance back above the low-water mark at $(date -u +%FT%TZ)." fi + # The spending wallet should never go DOWN, so a fall is worth more than + # a floor. Everything that spends from it also settles into it + # (SELF_FUNDING_SLUGS), and every execution tier charges more than it can + # spend, so barring a manual withdrawal the balance only rises. A + # low-water alarm fires after the money is gone; this fires on the first + # unexplained dollar. A withdrawal trips it too, deliberately: the alarm's + # job is to say "this wallet fell and nobody told me", and a human who + # withdrew can close the issue in one click. + - name: Upstream buyer wallet trend (a self-funding wallet that FALLS is unexplained) + if: always() + continue-on-error: true + run: | + TREND=$(curl -s --max-time 15 "$PROD/api/gateway-status" | jq -r '.upstreamBuyer.trend // "unknown"' 2>/dev/null || echo unknown) + echo "upstream buyer trend: $TREND" + TITLE="Upstream buyer wallet is DRAINING (unexplained fall)" + OPEN=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ "$TREND" = "draining" ]; then + if [ -z "$OPEN" ]; then + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body "The x402 upstream spending wallet has fallen below its high-water mark across several consecutive reads. + + That wallet is SELF-FUNDING: every tool that spends from it also settles into it, and every execution tier charges more than it can spend (worst case +\$0.005 / +\$0.01 / +\$0.05 / +\$0.30 per call). Its balance should only rise. A sustained fall means one of: + + 1. A manual withdrawal - close this issue if that was you. + 2. Upstream spend whose revenue never arrived: a buyer's payment verified and then failed to settle, which is the drain the per-payer ceiling in src/external-spend-guard.js bounds. Check /__operator/stats and the route-execute receipts. + 3. Something we do not understand, which is why this alarm exists. + + Balances are deliberately not published on /api/gateway-status; read the wallet directly to see the number." + else + gh issue comment "$OPEN" --repo "$GITHUB_REPOSITORY" --body "Still falling at $(date -u +%FT%TZ)." + fi + elif [ "$TREND" = "ok" ] && [ -n "$OPEN" ]; then + gh issue close "$OPEN" --repo "$GITHUB_REPOSITORY" --comment "Recovered: the wallet set a new high-water mark at $(date -u +%FT%TZ)." + fi + # Settlement freshness — the alarm that did not exist on 2026-08-07. # # The daily paid canary stopped buying on 2026-08-02 and reported success diff --git a/CLAUDE.md b/CLAUDE.md index 8f9a448a..43f976b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -332,6 +332,62 @@ with `res.statusCode === 200`. (`node_modules/@x402/express/dist/esm/index.mjs`. which is real but lives on the stdio npm package). Mutation-tested: removing `request_tool` from the listing fails 2 assertions, a fake tool name fails 1, a fake route fails 1. +- **Canary gate + settlement freshness alarm (2026-08-07):** the daily paid canary + stopped buying on **2026-08-02** and reported success every run for five days. Its + gate asked GitHub for the last SUCCESSFUL RUN, but a run whose gate SKIPS the buy + also concludes green, so every skip refreshed the window the next gate read and it + ratcheted permanently shut (measured across 40 runs: not one scheduled run bought + after the gate shipped; every real purchase came from a manual dispatch, which + bypasses the gate via `if: github.event_name == 'schedule'`). Nothing paged, because + skipping is not a failure — the ONLY surface that noticed was `/status`, reporting + the settlement component stale. **The gate now asks PRODUCTION when a canary last + BOUGHT** (`/api/status` settlement observation, written only by a canary that ran), + requiring fresh AND operational; unreachable status or a missing observation proceeds + with the buy, and every `jq` read carries a fallback because jq exits non-zero on a + non-JSON body and `set -e` would fail the gate. The canary job's `if` gained + `!cancelled()`: a job-level `if` with no status function still carries the implicit + `success()` on `needs`, so a FAILED gate would have SKIPPED the buy — the opposite of + what the comment beside it claimed, and never verified. **`heartbeat.yml` now pages on + a stale settlement observation** and self-heals once per episode by dispatching the + canary on FIRST detection only (a dispatch always buys; page rather than loop if + buying is genuinely broken). Proven end-to-end 2026-08-07: alarm fired → dispatched → + found a real failure → opened issues; then the 14:17 UTC SCHEDULED run bought (first + since 08-02) and the recovery branch closed its own issue. `scripts/test-canary-coverage.js` + locks the class: the gate must read `/api/status` and must NOT read `gh run list`, + every jq read must have a fallback, and the `if` must carry a status function. +- **Facilitator failure diagnostics (`src/facilitator-diagnostics.js`, 2026-08-07):** + 15 settle failures across Base/Solana/Polygon/Arbitrum all logged 200 characters of + `Coinbase…` — `@x402/core`'s `responseExcerpt` truncates an + error body at 200 chars, and on an HTML page that budget is spent entirely on markup. + A facilitator outage and an edge REFUSING OUR EGRESS were indistinguishable, and those + need opposite responses (wait vs build the fifth relay — Yahoo/Nasdaq/Sei/Nodely are + the existing four, and Nodely 403s Railway's IP outright). A global-fetch wrapper, + scoped to registered facilitator hosts and non-2xx non-JSON responses only, reads the + body BEFORE the vendor truncates it, strips markup, and classifies: cloudflare + challenge/block, access denied, rate limited, origin error behind the edge, gateway + timeout — keeping `cf-ray`/`server`/`retry-after`. It **clones** before reading + (consuming the body would break settlement), swallows every internal failure, and logs + once at boot so a silent failure to install is visible immediately. **Errors are also + LABELLED with the facilitator that threw them** (`labelFacilitatorErrors`): the failure + hooks log the chain and never the client, so Solana/Polygon/Arbitrum failures read as + Coinbase's words though the boot log routes those to PayAI and only Base to CDP — + clients are tried in order, so the surfacing error is the FIRST tried, not the chain's + owner. The label is **PREFIXED, never substituted**: `isPreBroadcastSettleRejection` + matches `settle failed (402)` as a substring, so replacing the message would silently + break the fallback's safety classification. `scripts/test-facilitator-diagnostics.js` + (30 assertions, offline, in CI). +- **Redis has REAL coverage in CI (2026-08-07):** nothing had ever connected to a redis. + `test-shared-limit.js` injects a fake store on purpose (it proves "two callers share + one counter", and a fake proves that exactly), which left the CLIENT path untested — + so a redis 4→6 bump arrived with a green CI that could not have caught a client + regression, the same worthless green as the tesseract 5→7 trap. Prod is **NOT** + in-memory (verified against Railway: `REDIS_URL` and `RATE_LIMIT_REPLICAS` are set, + and the shared limiter FAILS CLOSED). The test job now runs a `redis:7-alpine` service + container and `scripts/test-redis-integration.js` drives the real client (cap-of-1, + over-limit decrement, refund flooring, cache round trip). It asserts `degraded === false` + so it cannot pass via the fail-closed path with no server, and it **exits 1 rather than + skipping** when `REDIS_URL` is absent — a skipped integration test is why this went + untested at all. - **Marketplace latency / snapshot caching (`src/x402-economy.js`):** `GET /marketplace` (and `/api/x402-economy`) render from `x402EconomySnapshot()` — a ~500ms on-chain read (EIP-3009 USDC settlements on Base via CDP SQL). It is **stale-while-revalidate**: a fresh diff --git a/scripts/test-buyer-balance-trend.js b/scripts/test-buyer-balance-trend.js new file mode 100644 index 00000000..11bb65cc --- /dev/null +++ b/scripts/test-buyer-balance-trend.js @@ -0,0 +1,78 @@ +// The spending wallet should never go down, so a FALL is the signal. +// +// Everything that spends from it settles into it (SELF_FUNDING_SLUGS), and every +// execution tier charges more than it can spend. Barring a manual withdrawal the +// balance is monotonically non-decreasing. A low-water alarm fires after the +// money is gone; this fires on the first unexplained dollar. +// +// The hard part is the TRANSIENT dip that settlement ordering guarantees: we pay +// the seller during the handler and collect afterwards. An alarm that cannot +// tell that from a drain would page on every healthy call. +import { noteBuyerBalance } from "../src/tools/blockscout-kit.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; +const fresh = (v) => noteBuyerBalance(v, { reset: true }); + +{ + ok(fresh(10) === "ok", "the first read is a baseline, never an alarm"); + ok(noteBuyerBalance(10.3) === "ok", "a rise is healthy - this is what a self-funding wallet does"); + ok(noteBuyerBalance(10.6) === "ok", "…and keeps re-baselining upward"); +} + +{ + // THE FALSE POSITIVE THIS MUST NOT HAVE. Settlement runs after the handler, + // so the balance dips while a call is in flight and recovers when revenue + // lands. Paging on that would make the alarm useless within a day. + fresh(10); + ok(noteBuyerBalance(9.7) === "ok", "a dip inside tolerance is an in-flight call, not a drain"); + ok(noteBuyerBalance(10.4) === "ok", "and it recovers when the buyer's payment settles"); +} + +{ + // THE REAL DRAIN. A fall past tolerance, sustained across consecutive reads. + fresh(10); + ok(noteBuyerBalance(7) === "ok", "one big fall is not yet an alarm - a single read could be an in-flight max-tier call"); + ok(noteBuyerBalance(6.9) === "ok", "two is still not"); + ok(noteBuyerBalance(6.8) === "draining", "three consecutive reads below the high-water mark is a drain"); +} + +{ + // A slow bleed sits INSIDE tolerance on every individual read. If a + // within-tolerance read cleared the counter, a wallet losing $0.40 a read + // would never alarm - it would just quietly empty. + fresh(10); + noteBuyerBalance(6.5); // past tolerance, counter 1 + noteBuyerBalance(6.4); // counter 2 + ok(noteBuyerBalance(9.6) === "ok", + "a read within tolerance of the high-water mark does not itself alarm"); + fresh(10); + ok([9.6, 9.55, 9.5].map((v) => noteBuyerBalance(v)).every((s) => s === "ok"), + "…and a genuinely small wobble never alarms on its own"); +} + +{ + // Recovery must clear it, or one bad afternoon pages forever. + fresh(10); + noteBuyerBalance(6); noteBuyerBalance(5.9); + ok(noteBuyerBalance(5.8) === "draining", "draining while it is falling"); + ok(noteBuyerBalance(11) === "ok", "a new high clears the alarm and re-baselines"); + ok(noteBuyerBalance(10.9) === "ok", "and the counter really was reset, not merely masked"); +} + +{ + ok(fresh(NaN) === "unknown", "an unreadable balance is unknown, never a drain"); + ok(noteBuyerBalance(undefined) === "unknown", "and never throws on junk"); +} + +{ + // A withdrawal looks exactly like a drain, and SHOULD: the alarm's job is to + // say "this wallet fell and nobody told me". A human who withdrew can close + // the issue; a silent fall is the one we must never miss. + fresh(50); + const seq = [20, 20, 20].map((v) => noteBuyerBalance(v)); + ok(seq[2] === "draining", "a manual withdrawal alarms too - indistinguishable on purpose"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/test-external-spend-guard.js b/scripts/test-external-spend-guard.js new file mode 100644 index 00000000..75ae0192 --- /dev/null +++ b/scripts/test-external-spend-guard.js @@ -0,0 +1,147 @@ +// Offline tests for src/external-spend-guard.js. +// +// The hole it closes: settlement runs AFTER the handler, and the external +// routing handler pays a third-party seller from our wallet. A payment that +// VERIFIES and then fails to SETTLE leaves us out the upstream spend with the +// buyer charged nothing. Self-dealt - one wallet listing the seller and buying +// from it - every drained dollar returns to the attacker. +import { + maySpend, noteSpend, resolveSpend, payerExposureUsd, exposureSnapshot, __reset, +} from "../src/external-spend-guard.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; + +const A = "0xAbCdEf0123456789AbCdEf0123456789AbCdEf01"; + +// --- the attack, in one block ------------------------------------------------ +{ + __reset(); + // An explicit ceiling, not the shipped default: this block is about the + // MECHANISM, and tying it to whatever the default happens to be today would + // make it pass for the wrong reason the next time a tier moves. + const CEIL = { maxUnsettledUsd: 0.75 }; + // Call 1: allowed, we spend upstream. + ok(maySpend(A, 0.5, CEIL).ok, "a fresh payer may spend"); + const h1 = noteSpend(A, 0.5); + ok(payerExposureUsd(A) === 0.5, "the spend counts as exposure while unresolved"); + + // Their payment FAILS to settle. This is the whole point: handler success is + // not revenue, and the exposure must survive it. + resolveSpend(h1, false); + ok(payerExposureUsd(A) === 0.5, + `an UNSETTLED spend keeps counting against the payer (got ${payerExposureUsd(A)})`); + + // Call 2 from the same wallet is refused before we spend a second time. + const second = maySpend(A, 0.5, CEIL); + ok(second.ok === false, "a second call is refused while the first is unpaid - the drain stops at one"); + ok(/has not settled/i.test(second.reason), `the refusal explains itself (got: ${second.reason})`); +} + +{ + // The honest buyer's path: settle, and exposure clears immediately. + __reset(); + const h = noteSpend(A, 0.5); + resolveSpend(h, true); + ok(payerExposureUsd(A) === 0, "a SETTLED spend clears the exposure at once"); + ok(maySpend(A, 0.5, { maxUnsettledUsd: 0.75 }).ok, "and the payer may immediately spend again - this is a debt ceiling, not a reputation"); +} + +{ + // A wallet that pays reliably is never impeded, however many calls it makes. + __reset(); + let everRefused = false; + for (let i = 0; i < 25; i++) { + if (!maySpend(A, 0.5).ok) everRefused = true; + resolveSpend(noteSpend(A, 0.5), true); + } + ok(!everRefused && payerExposureUsd(A) === 0 && maySpend(A, 0.5).ok, + "25 settled calls in a row are never refused and leave zero exposure - a good buyer never hits the ceiling"); +} + +// --- identity handling ------------------------------------------------------- +{ + __reset(); + const h = noteSpend(A.toLowerCase(), 0.5); + resolveSpend(h, false); + ok(payerExposureUsd(A.toUpperCase().replace("0X", "0x")) === 0.5, + "EVM addresses are case-insensitive, so a payer cannot reset their ledger by changing case"); + + // base58 / Stellar / Algorand are case-SENSITIVE: folding them merges + // distinct payers, the same rule src/payer.js enforces. + const s1 = "GDNJXCKW7ZM7GEEVP674TWPU26YJNBQ2FI4ZIPRKTPTNUEJMDHFJWWRL"; + const h2 = noteSpend(s1, 0.4); + resolveSpend(h2, false); + ok(payerExposureUsd(s1.toLowerCase()) === 0, + "a base58/Stellar address is NOT case-folded - folding would merge distinct payers"); +} + +{ + __reset(); + // An unattributable payer (free mode, a rail whose payer we cannot read) is + // allowed: refusing would break every legitimate buyer on those rails, and a + // single call is still bounded by the tier cap. + const v = maySpend(null, 0.5); + ok(v.ok === true && /not attributable/i.test(v.reason), + "an unreadable payer is allowed and says why - the tier cap still bounds the call"); + ok(noteSpend(null, 0.5) === null, "…and nothing is recorded for a payer we cannot name"); +} + +// --- ceiling arithmetic ------------------------------------------------------ +{ + __reset(); + ok(maySpend(A, 0.5, { maxUnsettledUsd: 0.5 }).ok, "exactly at the ceiling is allowed"); + const h = noteSpend(A, 0.5); + resolveSpend(h, false); + ok(maySpend(A, 0.01, { maxUnsettledUsd: 0.5 }).ok === false, + "one cent past the ceiling is refused - the check is on the TOTAL, not the single call"); +} + +{ + // An unresolved row must not bar a payer forever (a process restart, a + // response that never finished), but must not clear so fast a loop outruns it. + __reset(); + const now = 1_000_000; + noteSpend(A, 0.5, now); + ok(payerExposureUsd(A, now + 60_000) === 0.5, "exposure stands a minute later"); + ok(payerExposureUsd(A, now + 11 * 60_000) === 0, "an unresolved spend ages out after the stale window"); +} + +{ + __reset(); + const h = noteSpend(A, 0.25); + resolveSpend(h, false); + const snap = exposureSnapshot(); + ok(snap.length === 1 && snap[0].unsettledUsd === 0.25 && snap[0].calls === 1, + `the operator view reports who owes upstream spend (got ${JSON.stringify(snap)})`); +} + +{ + // resolveSpend must never throw on junk - it runs inside a response + // finish handler, where an exception would break the response. + __reset(); + let threw = null; + try { resolveSpend(null, true); resolveSpend({ payer: "nope", id: 9 }, true); resolveSpend(undefined, false); } + catch (e) { threw = e; } + ok(!threw, "resolving an unknown or missing handle is a no-op, never a throw"); +} + +// --- the coupling that would silently kill a tier ---------------------------- +// If the ceiling is ever smaller than the largest execution tier's underlying +// cap, that tier is dead on arrival: a single legitimate call exceeds the +// ceiling and EVERY payer is refused, honest ones included. Nothing else would +// report this - the tier would simply never succeed - so it gets an assertion +// rather than the comment it started as. +{ + __reset(); + const { EXEC_TIERS } = await import("../src/tools/route-execute.js"); + const { __config } = await import("../src/external-spend-guard.js"); + const biggest = Math.max(...EXEC_TIERS.map((t) => t.underlyingMaxUsd)); + ok(__config.DEFAULT_MAX_UNSETTLED_USD >= biggest, + `the unsettled ceiling ($${__config.DEFAULT_MAX_UNSETTLED_USD}) covers the largest tier's underlying cap ($${biggest}) - otherwise that tier can never run`); + ok(maySpend("0x1111111111111111111111111111111111111111", biggest).ok, + "a single largest-tier call is allowed for a payer with no exposure"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/test-facilitator-diagnostics.js b/scripts/test-facilitator-diagnostics.js index c01b061f..d91b4050 100644 --- a/scripts/test-facilitator-diagnostics.js +++ b/scripts/test-facilitator-diagnostics.js @@ -187,5 +187,30 @@ const CF_BLOCK = ` Coinbase ${secretish}` }); + ok(!line.includes("abababab"), "a long hex run is redacted out of the logged excerpt"); + ok(!line.includes("pppppppppppp"), "a three-segment bearer-token shape is redacted too - the shape a plain long-run rule walks past"); + ok(line.includes("[redacted]"), "…and the redaction is visible rather than silent"); + ok(/Error for key/.test(line), "the surrounding words survive - the diagnosis is still readable"); + // A Cloudflare ray id is short and must NOT be swallowed by the redaction. + const ray = describeErrorResponse({ url: "https://f.example/settle", status: 502, headers: { "cf-ray": "8f2c1d4e5a6b7c8d-ATL" }, body: "Access denied" }); + ok(ray.includes("8f2c1d4e5a6b7c8d-ATL"), "a ray id survives redaction - it is the thing a provider asks for"); +} + console.log(`\n${pass} passed, ${fail} failed`); process.exit(fail ? 1 : 0); diff --git a/scripts/test-route-execute.js b/scripts/test-route-execute.js index 9614674b..ca238872 100644 --- a/scripts/test-route-execute.js +++ b/scripts/test-route-execute.js @@ -192,7 +192,32 @@ await expectErr({ slug: "broken-tool", params: {} }, 422, "underlying tool 422 p ok(routeExecuteHint(0.04)?.tool === "route-execute-plus", "$0.04 → plus tier boundary inclusive"); ok(routeExecuteHint(0.05)?.tool === "route-execute-max", "$0.05 → max tier (just over the plus cap)"); ok(routeExecuteHint(0.12)?.tool === "route-execute-max", "$0.12 → route-execute-max tier"); - ok(routeExecuteHint(0.9) === null, "$0.90 → no tier (above max)"); + // The pro tier (2026-08-07) exists because the $0.50 ceiling made the whole + // premium half of the index unroutable - the seller who reported the + // price:null bug prices their gates at $0.99 to $2.99, every one above the + // old max, so the router could only 409 them to their own direct route. + ok(routeExecuteHint(0.9)?.tool === "route-execute-pro", "$0.90 → route-execute-pro (was unroutable before the pro tier)"); + ok(routeExecuteHint(2.99)?.tool === "route-execute-pro", "$2.99 → pro tier - the real seller price that motivated it"); + ok(routeExecuteHint(3.0)?.tool === "route-execute-pro", "$3.00 → pro tier boundary inclusive"); + ok(routeExecuteHint(3.01) === null, "$3.01 → no tier: the ceiling still exists, it just moved"); + // The fee stays proportional rather than punishing size: 10% at the cap, the + // same spread as the max tier, not the 27x markup the plus tier was added to fix. + const pro = routeExecuteHint(3.0); + ok(Math.abs(pro.routingFeeUsd - 0.3) < 1e-9, `pro tier's fee at the cap is $0.30, a 10% spread (got ${pro.routingFeeUsd})`); +} + + +// --- the spending wallet's alarm must cover the biggest call ------------------ +// $0.50 was right when only Blockscout ($0.002/call) spent from that wallet. +// A tier that can spend $3.00 in one call makes "ok" mean "has at least $0.50" +// for a wallet that cannot cover a single call - the alarm would stay green +// right up to the failure it exists to prevent. Nothing else reports this. +{ + const { BUYER_LOW_DEFAULT_USD } = await import("../src/tools/blockscout-kit.js"); + const { EXEC_TIERS: TIERS } = await import("../src/tools/route-execute.js"); + const biggest = Math.max(...TIERS.map((t) => t.underlyingMaxUsd)); + ok(BUYER_LOW_DEFAULT_USD >= biggest, + `the upstream buyer low-water default ($${BUYER_LOW_DEFAULT_USD}) covers the largest tier's underlying spend ($${biggest}) - otherwise "ok" can mean "cannot fund one call"`); } console.log(`\n${pass} passed, ${fail} failed`); diff --git a/scripts/test-x402-live-quote.js b/scripts/test-x402-live-quote.js new file mode 100644 index 00000000..f5876f8e --- /dev/null +++ b/scripts/test-x402-live-quote.js @@ -0,0 +1,110 @@ +// Offline tests for src/x402-live-quote.js. +// +// Fixtures are the REAL shapes, captured from a live seller on 2026-08-07 who +// reported 39 endpoints indexed at price:null while every one of them returns a +// textbook 402. If these assertions pass against anything less than that real +// challenge, they are not testing what went wrong. +import { acceptsFromLive402, quoteFromAccepts, probeMethodsFor, isQuoteResponse } from "../src/x402-live-quote.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { if (c) { pass++; console.log(`ok - ${m}`); } else { fail++; console.error(`FAIL - ${m}`); } }; + +// The accepts entry exactly as the reporting seller's 402 carries it: $0.99 as +// 990000 atomic units of Base USDC, with the name in `extra`. +const REAL_ACCEPT = { + scheme: "exact", + network: "eip155:8453", + asset: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + amount: "990000", + payTo: "0x0bac88e8B47D9F2dC38E66dB9dA4b41032d24065", + maxTimeoutSeconds: 60, + extra: { name: "USDC", version: "2" }, +}; +const HEADER = Buffer.from(JSON.stringify({ x402Version: 2, accepts: [REAL_ACCEPT] })).toString("base64"); + +// --- reading the challenge --------------------------------------------------- +{ + ok(acceptsFromLive402({ header: HEADER })?.[0]?.amount === "990000", + "accepts are read from the base64 payment-required HEADER (x402 v2's home for them)"); + + // The same seller ALSO nests it in the body under `payment`. A reader that + // only understood a top-level `accepts` would have missed it. + const body = JSON.stringify({ error: { code: "PAYMENT_REQUIRED" }, payment: { rail: "cdp_x402", accepts: [REAL_ACCEPT] } }); + ok(acceptsFromLive402({ body })?.[0]?.payTo === REAL_ACCEPT.payTo, + "accepts are read from a body that NESTS them under `payment`"); + + ok(acceptsFromLive402({ body: JSON.stringify({ accepts: [REAL_ACCEPT] }) })?.length === 1, + "a plain top-level accepts body still works"); + + ok(acceptsFromLive402({ header: "not-base64-at-all", body: JSON.stringify({ accepts: [REAL_ACCEPT] }) })?.length === 1, + "an undecodable header falls through to the body rather than giving up"); + + ok(acceptsFromLive402({ header: "", body: "nope" }) === null, + "an unreadable challenge is null - never an empty-but-truthy quote"); + ok(acceptsFromLive402({}) === null, "nothing in, null out"); +} + +// --- pricing ----------------------------------------------------------------- +{ + const q = quoteFromAccepts([REAL_ACCEPT]); + ok(q.price === 0.99, `990000 atomic USDC prices as $0.99, not 990000 (got ${q.price})`); + ok(q.networks.includes("eip155:8453"), "the network travels with the quote"); + ok(q.payTo === REAL_ACCEPT.payTo, "the payTo is captured so the router can check where money goes"); + + // Base USDC wins over another chain, matching bazaarItemToTool's order, so a + // live-probed row and a Bazaar row are comparable. + const multi = quoteFromAccepts([ + { network: "solana:x", amount: "5000000", payTo: "sol", extra: { name: "USDC" } }, + REAL_ACCEPT, + ]); + ok(multi.price === 0.99 && multi.network === "eip155:8453", + `Base USDC is preferred when several chains are offered (got ${multi.price} on ${multi.network})`); + ok(multi.networks.length === 2, "every offered chain is still recorded"); +} + +{ + // THE MONEY ASSERTION. An asset we cannot price must not become $0 - that + // would publish a paid tool as free, and "free" is the one wrong answer a + // buyer acts on immediately. + const unknown = quoteFromAccepts([{ network: "eip155:8453", amount: "12345", payTo: "0xabc", extra: { name: "WEIRDTOKEN" } }]); + ok(unknown.price === null, `an unpriceable asset yields null, never 0 (got ${unknown.price})`); + ok(unknown.networks.includes("eip155:8453"), + "…but the network is still recorded, so the row reads payable-on-Base rather than unknown"); + + const explicit = quoteFromAccepts([{ network: "eip155:1", amount: "1500000000000000000", payTo: "0x1", extra: { name: "DAI", decimals: 18 } }]); + ok(explicit.price === 1.5, `an explicit decimals hint is honoured (got ${explicit.price})`); + + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "-5", extra: { name: "USDC" } }]).price === null, + "a negative amount is corrupt, not free"); + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "abc", extra: { name: "USDC" } }]).price === null, + "a non-numeric amount does not become NaN dollars"); + ok(quoteFromAccepts([{ network: "eip155:8453", amount: "0", extra: { name: "USDC" } }]).price === 0, + "an explicit zero IS free and is reported as such"); + ok(quoteFromAccepts([]) === null && quoteFromAccepts(null) === null, "no accepts, no quote"); +} + +// --- which methods to probe -------------------------------------------------- +{ + // The defect in one assertion: the reporting seller's routes 404 on GET and + // 402 on POST. A GET-only prober sees a dead catalogue. + ok(probeMethodsFor({ method: "GET", methodInferred: true }).includes("POST"), + "an INFERRED GET still tries POST - the seller whose routes 404 on GET and 402 on POST"); + ok(probeMethodsFor({}).join(",") === "GET,POST", "no stated method tries both, GET first (cheapest)"); + ok(probeMethodsFor({ method: "POST" }).join(",") === "POST", "a stated POST is taken at its word"); + ok(probeMethodsFor({ method: "GET", methodInferred: false }).join(",") === "GET,POST", + "a stated GET still falls back to POST, because a 404 costs one request and a missed catalogue costs a seller"); + + // An unpaid probe must never be able to mutate a stranger's server. + for (const m of ["PUT", "PATCH", "DELETE"]) { + ok(probeMethodsFor({ method: m }).length === 0, `${m} is never probed - an unpaid probe must not mutate anything`); + } +} + +{ + ok(isQuoteResponse(402) === true, "402 is the healthy answer to an unpaid call"); + ok(isQuoteResponse(200) === false, "200 means the route is not paywalled, which is not a quote"); + ok(isQuoteResponse(404) === false && isQuoteResponse(500) === false, "errors are not quotes"); +} + +console.log(`\n${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/src/external-spend-guard.js b/src/external-spend-guard.js new file mode 100644 index 00000000..7acd038c --- /dev/null +++ b/src/external-spend-guard.js @@ -0,0 +1,157 @@ +// Bound what a single buyer can make us spend upstream before they have paid us. +// +// THE HOLE. @x402/express runs the handler FIRST and settles AFTER, and a <400 +// response whose settlement then fails has its body discarded and returns 402. +// On the external routing path the handler PAYS A THIRD-PARTY SELLER from our +// spending wallet. So the sequence is: buyer's payment verifies, we pay the +// seller real USDC, our settlement fails, buyer is charged nothing. We are out +// the upstream spend with no revenue and no recourse. +// +// Verify-then-fail-to-settle is not hypothetical: it happens naturally when a +// payer's balance drops between the two (documented on Solana, where our own +// best buyer drained to $0 and its last four purchases "timed out"). Self-dealt +// it is an attack - the same wallet lists the seller and buys from it, so every +// drained dollar lands back in the attacker's pocket, bounded per call only by +// the tier cap. It scales with exactly the cap we want to raise. +// +// The existing guards bound WHAT we pay (canonical-USDC asset pin, tier cap +// re-checked against the live 402, a 50-settlement/3-payer reliability floor). +// None of them bound WHETHER WE GET PAID, because that is decided after the +// handler has already spent. +// +// So: a payer may carry only so much UNSETTLED upstream spend at once. Spend is +// recorded before the buy and resolved on the FINAL response - `res.on("finish")` +// with statusCode 200, i.e. after @x402/express has settled - which is the same +// rule the idempotency cache uses and for the same reason: a 200 is not revenue +// until settlement says so. +// +// Deliberately NOT a reputation system. It is a debt ceiling: pay for what you +// asked for and your exposure clears within seconds. It only ever bites a payer +// whose settlements are failing, which is the population it exists for. + +/** Max unsettled upstream spend one payer may carry, in USD. + * + * Sized at TWO top-tier calls (route-execute-pro covers an underlying $3.00), + * so a buyer can have a second call in flight while the first is still + * settling - agents pipeline, and refusing that would break honest traffic - + * while a wallet whose payments never settle is stopped after two rather than + * draining indefinitely. + * + * MUST be re-sized whenever a higher execution tier is added, or the new tier + * is dead on arrival: a single call larger than this ceiling is refused for + * every payer, including the honest ones. That coupling is the reason the + * tier table points back at this constant. */ +const DEFAULT_MAX_UNSETTLED_USD = Number(process.env.EXTERNAL_MAX_UNSETTLED_USD || 6.0); + +/** How long an unresolved spend counts against a payer. A settlement that never + * reports (process restart, a response that never finished) must not bar a + * payer forever, but it must not clear so fast that a fast loop outruns it. */ +const STALE_MS = Number(process.env.EXTERNAL_SPEND_STALE_MS || 10 * 60_000); + +/** Payers exempt from the ceiling: our own canary and any operator-listed + * wallet. Comma-separated, lowercased for EVM. */ +function exemptSet() { + return new Set( + String(process.env.EXTERNAL_SPEND_EXEMPT || "") + .split(",").map((s) => s.trim().toLowerCase()).filter(Boolean), + ); +} + +// payer -> [{ usd, at, settled }] +const ledger = new Map(); +let seq = 0; + +const keyOf = (payer) => { + if (typeof payer !== "string" || !payer.trim()) return null; + // EVM addresses are case-insensitive; base58/Stellar/Algorand are NOT, and + // folding them merges distinct payers (same rule as src/payer.js). + const p = payer.trim(); + return /^0x[0-9a-fA-F]{40}$/.test(p) ? p.toLowerCase() : p; +}; + +function prune(rows, now) { + return rows.filter((r) => !r.settled && now - r.at < STALE_MS); +} + +/** Current unsettled exposure for a payer, in USD. */ +export function payerExposureUsd(payer, now = Date.now()) { + const k = keyOf(payer); + if (!k) return 0; + const rows = prune(ledger.get(k) || [], now); + if (rows.length) ledger.set(k, rows); else ledger.delete(k); + return rows.reduce((s, r) => s + r.usd, 0); +} + +/** + * May this payer make us spend `usd` upstream right now? + * + * An UNKNOWN payer (free mode, a non-EIP-3009 rail we cannot attribute) is + * allowed: refusing there would break every legitimate buyer on a rail whose + * payer we cannot read, and the tier cap still bounds a single call. This is a + * per-payer debt ceiling, not an identity requirement. + */ +export function maySpend(payer, usd, { maxUnsettledUsd = DEFAULT_MAX_UNSETTLED_USD, now = Date.now() } = {}) { + const k = keyOf(payer); + if (!k) return { ok: true, reason: "payer not attributable - bounded by the tier cap alone" }; + if (exemptSet().has(k.toLowerCase())) return { ok: true, reason: "exempt" }; + const exposure = payerExposureUsd(k, now); + const next = exposure + (Number(usd) || 0); + if (next > maxUnsettledUsd) { + return { + ok: false, + exposure, + reason: + `this wallet already has $${exposure.toFixed(3)} of upstream spend from earlier calls whose payment has not settled` + + ` (ceiling $${maxUnsettledUsd}). It clears as soon as those settle.`, + }; + } + return { ok: true, exposure }; +} + +/** Record an upstream spend as UNSETTLED. Returns a handle to resolve later. */ +export function noteSpend(payer, usd, now = Date.now()) { + const k = keyOf(payer); + if (!k) return null; + const row = { id: ++seq, usd: Number(usd) || 0, at: now, settled: false }; + const rows = prune(ledger.get(k) || [], now); + rows.push(row); + ledger.set(k, rows); + return { payer: k, id: row.id }; +} + +/** + * Resolve a recorded spend once the FINAL response is known. + * + * `settled` must come from the post-settlement status (res.on("finish") with + * statusCode === 200), never from the handler's own return: the handler + * succeeding is precisely the state that precedes a settlement failure. + */ +export function resolveSpend(handle, settled) { + if (!handle?.payer) return; + const rows = ledger.get(handle.payer); + if (!rows) return; + const row = rows.find((r) => r.id === handle.id); + if (!row) return; + if (settled) { + row.settled = true; + const left = rows.filter((r) => !r.settled); + if (left.length) ledger.set(handle.payer, left); else ledger.delete(handle.payer); + } + // NOT settled: the row stays as exposure until it ages out. That is the + // whole point - an unpaid spend must keep counting against the payer. +} + +/** Operator view: who currently owes us upstream spend. Counts only. */ +export function exposureSnapshot(now = Date.now()) { + const out = []; + for (const [payer, rows] of ledger) { + const live = prune(rows, now); + if (!live.length) continue; + out.push({ payer, unsettledUsd: Number(live.reduce((s, r) => s + r.usd, 0).toFixed(6)), calls: live.length }); + } + return out.sort((a, b) => b.unsettledUsd - a.unsettledUsd); +} + +/** Test-only. */ +export function __reset() { ledger.clear(); seq = 0; } +export const __config = { DEFAULT_MAX_UNSETTLED_USD, STALE_MS }; diff --git a/src/facilitator-diagnostics.js b/src/facilitator-diagnostics.js index c7197621..5500f38c 100644 --- a/src/facilitator-diagnostics.js +++ b/src/facilitator-diagnostics.js @@ -91,7 +91,19 @@ export function describeErrorResponse({ url, status, headers, body } = {}) { .filter(Boolean); // 600 chars of WORDS, having spent none of the budget on markup. The page // that started this fits its whole message inside it. - const excerpt = text.length > 600 ? `${text.slice(0, 597)}...` : text; + // + // Long hex/base64 runs are REDACTED first. We are logging a body we did not + // write, from a host we authenticate to, straight into a log aggregator - and + // an error page that echoes a request header or a signed payload would put a + // credential there permanently. Nothing diagnostic is lost: a Cloudflare ray + // id is short, and no failure has ever been explained by a 64-character blob. + const redacted = text + // JWT-shaped first: its segments are dot-separated and individually short, + // so a plain "long run" rule walks straight past a whole bearer token. + .replace(/\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[redacted]") + .replace(/\b(?:0x)?[0-9a-fA-F]{32,}\b/g, "[redacted]") + .replace(/\b[A-Za-z0-9+/_-]{40,}={0,2}\b/g, "[redacted]"); + const excerpt = redacted.length > 600 ? `${redacted.slice(0, 597)}...` : redacted; return `[facilitator-diag] ${status} from ${url} - ${verdict}` + `${bits.length ? ` | ${bits.join(" ")}` : ""}` + `${excerpt ? ` | body: ${excerpt}` : " | body: "}`; diff --git a/src/payments.js b/src/payments.js index 1353fa0f..595cb011 100644 --- a/src/payments.js +++ b/src/payments.js @@ -320,11 +320,11 @@ export const isIdentityBoundRoute = (def) => // Router tiers only - the subset whose Algorand revenue is chain-matched to // the AVM spending wallet (see avmPayToFor). Blockscout stays out: its // upstream spend is Base-pinned regardless of the buyer's rail. -export const AVM_SELF_FUNDING_SLUGS = new Set(["route-execute", "route-execute-plus", "route-execute-max"]); +export const AVM_SELF_FUNDING_SLUGS = new Set(["route-execute", "route-execute-plus", "route-execute-max", "route-execute-pro"]); const AVM_UPSTREAM_BUYER_ADDRESS = (process.env.ALGORAND_UPSTREAM_BUYER_ADDRESS || "").trim(); export const SELF_FUNDING_SLUGS = new Set([ - "route-execute", "route-execute-plus", "route-execute-max", + "route-execute", "route-execute-plus", "route-execute-max", "route-execute-pro", // Blockscout kit (2026-07-29, the house rule: everything that spends from the // burner settles to the burner): each call pays Blockscout ~$0.002 upstream // from the same wallet, so treasury-settled revenue was a slow one-way diff --git a/src/server.js b/src/server.js index 75a468fa..6c572104 100644 --- a/src/server.js +++ b/src/server.js @@ -24,6 +24,7 @@ import { PERSISTENT as memoryPersistent, } from "./tools/memory.js"; import { payerFromRequest, payerFromPaymentResponse } from "./payer.js"; +import { resolveSpend as resolveExternalSpend } from "./external-spend-guard.js"; import { registerWellKnown, removeWellKnown, getWellKnown, listWellKnown } from "./well-known-store.js"; import { backupPlan, backupStatus, runBackup, startBackupScheduler } from "./backup.js"; import { assertAvmValidityCovers } from "./avm-validity.js"; @@ -4474,6 +4475,21 @@ for (const tool of ALL_KIT) { const result = await tool.handler(input, req); + // A handler that spent real money upstream (external route-execute) leaves + // a handle on the request. Resolve it against the FINAL response, not the + // handler's return: settlement runs after this, and handler-success is + // precisely the state that precedes a settlement failure. Only a settled + // 200 clears the payer's exposure; anything else leaves it standing, which + // is what stops a wallet whose payments never settle from draining the + // upstream wallet one call at a time. Same doctrine as the idempotency + // cache's commit-on-finish. + if (req.__externalSpend) { + const handle = req.__externalSpend; + res.on("finish", () => { + try { resolveExternalSpend(handle, res.statusCode === 200); } catch { /* never break a response */ } + }); + } + if (cachePolicy) { noteCacheOutcome(cacheKey ? "miss" : "skip"); res.setHeader("X-Cache", cacheKey ? "miss" : "skip"); diff --git a/src/tools/blockscout-kit.js b/src/tools/blockscout-kit.js index decc2033..1b6beba0 100644 --- a/src/tools/blockscout-kit.js +++ b/src/tools/blockscout-kit.js @@ -290,7 +290,66 @@ export const BLOCKSCOUT_TOOLS = [ // with graceful "unknown" (an RPC flake must never page). const BASE_RPCS = ["https://mainnet.base.org", "https://base.llamarpc.com", "https://base.drpc.org"]; const USDC_BASE = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913"; -const BUYER_LOW_USD = () => Number(process.env.UPSTREAM_BUYER_LOW_USD || "0.5"); +// Sized against the LARGEST single spend this wallet can be asked to make, not +// against the smallest. +// +// $0.50 was right when the only thing spending from here was Blockscout at +// $0.002/call: a wallet above it could serve hundreds of calls. Then +// route-execute-pro (2026-08-07) made a single call able to spend $3.00 +// upstream, and "ok" started meaning "has at least $0.50" for a wallet that +// could not cover one call. The alarm would have stayed green right up to the +// failure it exists to prevent. +// +// Two largest-tier calls, so we are paged with room to top up rather than at +// the moment of starvation. MUST be re-sized whenever a bigger execution tier +// lands - locked by an assertion in scripts/test-route-execute.js, because a +// threshold that quietly stops covering the biggest call reports nothing. +export const BUYER_LOW_DEFAULT_USD = 6; + +// THIS WALLET SHOULD NEVER GO DOWN, so a fall is worth more than a floor. +// +// Everything that spends from it also settles INTO it: SELF_FUNDING_SLUGS sets +// payTo to this address for exactly those tools (payments.js acceptsForItem), +// and every execution tier charges more than it can spend - worst case +$0.005, +// +$0.01, +$0.05, +$0.30 per call. Blockscout is the same shape ($0.002 upstream +// against a priced tool). So barring a manual withdrawal the balance is +// monotonically non-decreasing, and a SUSTAINED fall means something we do not +// understand is happening: the verify-then-fail-to-settle drain, a spend whose +// revenue never arrived, or a withdrawal nobody mentioned. +// +// A low-water alarm fires after the money is gone. This fires on the first +// unexplained dollar, which is the whole difference. +// +// It must tolerate a TRANSIENT dip, because settlement ordering guarantees one: +// we pay the seller during the handler and collect afterwards, so the balance +// is legitimately lower in between. Hence a high-water mark, a tolerance, and a +// requirement that the fall persist across consecutive reads (each 5 min apart) +// before it is called draining. +const BUYER_DROP_TOLERANCE_USD = Number(process.env.UPSTREAM_BUYER_DROP_TOLERANCE_USD || 0.5); +const BUYER_DROP_READS = Number(process.env.UPSTREAM_BUYER_DROP_READS || 3); +let buyerHighWater = null; +let buyerBelowReads = 0; + +/** Bucketed trend for the spending wallet: "ok" | "draining". Never a number - + * /api/gateway-status is public and balances stay off it. Exported for tests. */ +export function noteBuyerBalance(balance, { reset = false } = {}) { + if (reset) { buyerHighWater = null; buyerBelowReads = 0; } + if (!Number.isFinite(balance)) return "unknown"; + if (buyerHighWater == null || balance >= buyerHighWater) { + // A new high (or the first read) is the healthy case: re-baseline and clear. + buyerHighWater = balance; + buyerBelowReads = 0; + return "ok"; + } + if (buyerHighWater - balance <= BUYER_DROP_TOLERANCE_USD) { + // Within tolerance: an in-flight call, not a drain. Do NOT reset the + // counter - a slow bleed sits inside tolerance on every single read. + return buyerBelowReads >= BUYER_DROP_READS ? "draining" : "ok"; + } + buyerBelowReads += 1; + return buyerBelowReads >= BUYER_DROP_READS ? "draining" : "ok"; +} +const BUYER_LOW_USD = () => Number(process.env.UPSTREAM_BUYER_LOW_USD || String(BUYER_LOW_DEFAULT_USD)); const BUYER_STATUS_CACHE_MS = 5 * 60_000; let buyerStatusCache = null; /** Bucketed BALANCE of the Base spending wallet. Nothing more. @@ -334,7 +393,7 @@ export async function upstreamBuyerStatus() { } result = balance == null ? { configured: true, status: "unknown", attests: "balance-only" } - : { configured: true, status: balance < BUYER_LOW_USD() ? "low" : "ok", attests: "balance-only" }; + : { configured: true, status: balance < BUYER_LOW_USD() ? "low" : "ok", attests: "balance-only", trend: noteBuyerBalance(balance) }; } catch { result = { configured: true, status: "unknown", attests: "balance-only" }; } diff --git a/src/tools/route-execute.js b/src/tools/route-execute.js index 06d33bc9..c7f55497 100644 --- a/src/tools/route-execute.js +++ b/src/tools/route-execute.js @@ -17,7 +17,8 @@ // that names the tool and its direct route, so the buyer can call it at list // price instead. import { createHash } from "node:crypto"; -import { paymentHeaderOf } from "../payer.js"; +import { paymentHeaderOf, payerFromRequest } from "../payer.js"; +import { maySpend, noteSpend, resolveSpend } from "../external-spend-guard.js"; import { findTools } from "../find.js"; import { isIdentityBoundRoute } from "../payments.js"; @@ -41,6 +42,18 @@ export const EXEC_TIERS = [ // external inventory the seller review identified as routable demand. { slug: "route-execute-plus", execPriceUsd: 0.05, underlyingMaxUsd: 0.04 }, { slug: "route-execute-max", execPriceUsd: 0.55, underlyingMaxUsd: 0.5 }, + // 2026-08-07: the $0.50 ceiling made the whole premium half of the index + // unroutable. The seller who reported the price:null bug prices their gates + // at $0.99, $1.50 and $2.99 - every one of them above the top tier, so the + // router could only 409 them at their own direct route. Same 10% spread as + // the max tier, so the curve stays proportional rather than punishing size. + // + // This tier is only safe because of the per-payer debt ceiling in + // external-spend-guard.js. Settlement runs AFTER the handler, so raising the + // cap raises exactly one exposure: what a buyer whose payment verifies and + // then fails to settle can make us spend before we stop them. Keep + // EXTERNAL_MAX_UNSETTLED_USD sized against THIS number, not the old $0.50. + { slug: "route-execute-pro", execPriceUsd: 3.3, underlyingMaxUsd: 3.0 }, ]; const EXEC_SLUGS = new Set(EXEC_TIERS.map((t) => t.slug)); /** Which execution tier (if any) can run a tool at `underlyingUsd`, and the @@ -258,10 +271,38 @@ export function buildRouteExecuteTool({ getCatalog, baseUrl = "", tier = EXEC_TI extUrl = qs ? `${ext.url}${ext.url.includes("?") ? "&" : "?"}${qs}` : ext.url; extBody = undefined; } + // PER-PAYER DEBT CEILING. Everything above bounds WHAT we pay (the + // canonical-USDC asset pin, this tier's cap re-checked against the + // live 402, the 50-settlement reliability floor). Nothing bounds + // WHETHER WE GET PAID: settlement runs AFTER this handler, so a + // payment that verifies and then fails to settle leaves us having + // spent real USDC upstream for a buyer who is charged nothing. + // Self-dealt - one wallet listing the seller and buying from it - + // every drained dollar returns to the attacker, bounded per call only + // by `cap`. A 4xx here cancels the buyer's settlement, so refusing + // costs an honest buyer nothing. + const spendPayer = payerFromRequest(req); + const allowed = maySpend(spendPayer, extUsd); + if (!allowed.ok) throw bad(`External routing is paused for this wallet: ${allowed.reason}`, 429); + const spendHandle = noteSpend(spendPayer, extUsd); + // Handed to server.js on the REQUEST, because a tool handler is called + // as handler(input, req) and never receives `res`. The first draft + // registered res.on("finish") here, where `res` is undefined - a guard + // that installs nothing and reports no error, which is the exact shape + // of defect this session keeps finding. server.js resolves it on the + // FINAL response (post-settlement), never on handler success. + if (spendHandle && req && typeof req === "object") req.__externalSpend = spendHandle; let paid; try { paid = await payExternal(extUrl, { method: extMethod, body: extBody, maxAtomic: BigInt(Math.round(cap * 1e6)), chain }); } catch (e) { + // The exposure DELIBERATELY stands. It is tempting to clear it here + // ("the buy failed, so we never spent"), but payExternal can throw + // after signing and broadcasting - a network error on the response, + // a timeout - and clearing on those is exactly the case that lets a + // spend disappear from the ledger. It ages out on its own within + // the stale window, so an honest buyer caught by a seller outage + // waits, while a spend we cannot account for keeps counting. const sc = e?.statusCode && e.statusCode >= 400 && e.statusCode < 600 ? e.statusCode : 502; throw bad(`External seller "${ext.seller}" failed: ${String(e?.message || e).slice(0, 200)}`, sc); } diff --git a/src/x402-index.js b/src/x402-index.js index f0224761..29ecd0fc 100644 --- a/src/x402-index.js +++ b/src/x402-index.js @@ -36,6 +36,7 @@ import { fetchAllBazaarItems, isBazaarDiscoveryUrl } from "./bazaar-pager.js"; import { RAILS, railKey, truncateCaip2 } from "./rails.js"; import { CHAIN_PAGES, marketSellers } from "./market-page.js"; import { WELL_KNOWN_PATH, discoveryNote } from "./discovery-note.js"; +import { acceptsFromLive402, quoteFromAccepts, probeMethodsFor, isQuoteResponse } from "./x402-live-quote.js"; import { summarize, fmtUsd, fmtPct } from "./economy.js"; import { rankBy, canonicalHost } from "./leaderboard.js"; import { routeExecuteHint } from "./tools/route-execute.js"; @@ -1144,6 +1145,91 @@ function paywallProbeDue() { return paywallProbeCursor++ % Math.max(1, Math.ceil(cache.size / PAYWALL_PROBES_PER_CYCLE) || 1) === 0; } +// How many priceless routes we will quote-probe per seller per crawl. The +// crawl runs every 5 minutes across ~2,200 origins, so this is the difference +// between "we learn a catalogue's prices within the hour" and "we hammer a +// stranger's server". A route that gets priced is never probed again (it has a +// price); one that cannot be priced backs off through probeDue like every +// other path. See the #645 note below on why per-PATH backoff matters. +const LIVE_QUOTE_PROBES_PER_CRAWL = 3; +// GLOBAL ceiling per crawl CYCLE, not just per seller. Three per seller sounds +// gentle until you multiply: roughly a third of indexed rows carry no price, so +// a per-seller-only limit fires thousands of outbound requests every 5 minutes +// across the whole index - which is issue #645 rebuilt with a different label. +// Per-route backoff eventually quiets the sellers who never answer 402, but +// "eventually" is the first several cycles, and the seller feels those. This +// bounds the whole cycle; the rest simply wait their turn on the next one. +const LIVE_QUOTE_PROBES_PER_CYCLE = Number(process.env.LIVE_QUOTE_PROBES_PER_CYCLE || 60); +let liveQuoteBudget = LIVE_QUOTE_PROBES_PER_CYCLE; + +/** + * Learn price + networks from a live 402 for rows that have neither. + * + * THE DEFECT (reported by a seller, 2026-08-07): a manifest may list + * `resources` as bare URL strings, which carry no price, and probePaywall - + * the only thing that talks to a seller's endpoint - filters on + * `Number(t.price) > 0`. So a priceless row was never probed, and probing is + * the only thing that could have given it a price. Their 39 endpoints indexed + * at price:null while every one returned a textbook 402 on POST. Across the + * index that same day: 146 of 500 sellers had zero priced rows. + * + * Only ever ADDS information: a row that already has a price is skipped, and a + * probe that cannot produce a quote leaves the row exactly as it was. + */ +async function enrichLiveQuotes(tools, originUrl) { + if (!Array.isArray(tools) || !tools.length) return tools; + const candidates = tools.filter( + (t) => t + && typeof t.route === "string" && t.route.startsWith("/") + && t.seller !== LOCAL_SELLER // never probe ourselves + && !(Number(t.price) > 0) // already priced: nothing to learn + && !(Array.isArray(t.networks) && t.networks.length) // already payable-evidenced + && probeMethodsFor(t).length // never PUT/PATCH/DELETE + && probeDue(originUrl, `quote:${t.route}`), + ).slice(0, Math.max(0, Math.min(LIVE_QUOTE_PROBES_PER_CRAWL, liveQuoteBudget))); + if (!candidates.length) return tools; + liveQuoteBudget -= candidates.length; + + const { assertPublicUrl, ssrfDispatcher } = await import("./tools/fetch-guard.js"); + for (const tool of candidates) { + const target = `${originUrl}${tool.route}`; + let learned = null; + for (const method of probeMethodsFor(tool)) { + try { + // Crawled URLs are external data and could DNS-rebind between crawl and + // now: validate then pin, exactly as probePaywall does. + await assertPublicUrl(target); + const res = await fetch(target, { + method, + headers: { Accept: "application/json", ...(method === "POST" ? { "Content-Type": "application/json" } : {}) }, + ...(method === "POST" ? { body: "{}" } : {}), + dispatcher: ssrfDispatcher, + redirect: "manual", + signal: AbortSignal.timeout(8000), + }); + if (!isQuoteResponse(res.status)) continue; // 404 on GET is expected for a POST-only seller + // The quote lives in the header for x402 v2 and in the body for several + // real sellers; read a bounded slice of both and let the parser decide. + const body = await res.text().catch(() => ""); + const quote = quoteFromAccepts( + acceptsFromLive402({ header: res.headers.get("payment-required"), body: body.slice(0, 64_000) }), + ); + if (quote) { learned = { ...quote, method }; break; } + } catch { /* unreachable, blocked, or malformed - try the next method */ } + } + noteProbeOutcome(originUrl, `quote:${tool.route}`, Boolean(learned)); + if (!learned) continue; + // Price may be null for an asset we refuse to guess at; the networks alone + // still move the row from payable:"unknown" to payable:"x402", which is the + // honest and useful half of the answer. + if (learned.price != null && !(Number(tool.price) > 0)) tool.price = learned.price; + if (learned.networks?.length) tool.networks = [...new Set([...(tool.networks || []), ...learned.networks])]; + if (learned.method && learned.method !== tool.method) { tool.method = learned.method; tool.methodInferred = false; } + tool.quoteSource = "live-402"; + } + return tools; +} + async function probePaywall(tools) { // A cached tool row has NO `url` field — the callable URL is derived as // seller + route, the same way routeQuery builds it (see the `url:` mapping @@ -1308,6 +1394,9 @@ async function crawlSeller(originUrl) { // mergeManifestIntoTools for the 16 -> 30 regression that proved why. tools = mergeManifestIntoTools(normaliseManifestTools(manifest, originUrl), tools); tools = dropUnvouchedNonProductRoutes(tools, (bazaarToolsByOrigin.get(originUrl) || []).map((t) => t.route)); + // Learn prices the catalogue could not carry. Bounded per seller per crawl + // and backed off per route; only ever adds information. + tools = await enrichLiveQuotes(tools, originUrl); cache.set(originUrl, { manifest, @@ -1409,6 +1498,10 @@ async function crawlSeller(originUrl) { bazaarTools.map((t) => t.route) ); if (tools.length) { + // Same enrichment as the manifest path. A seller discovered through the + // FALLBACK surfaces is even less likely to have published a price, so + // skipping it here would leave the worst-served sellers unpriced. + await enrichLiveQuotes(tools, originUrl); // A real (non-synthesized) manifest from a past crawl is kept; a stale // synthesized one is rebuilt so a newly appeared openapi title wins. const keepManifest = prev?.manifest && !prev.manifest.synthesized ? prev.manifest : null; @@ -1607,6 +1700,7 @@ async function runCrawl() { crawlInFlight = true; try { const seeds = seedList(); + liveQuoteBudget = LIVE_QUOTE_PROBES_PER_CYCLE; // fresh allowance each cycle await runPool(seeds, CRAWL_CONCURRENCY, crawlSeller); } finally { crawlInFlight = false; diff --git a/src/x402-live-quote.js b/src/x402-live-quote.js new file mode 100644 index 00000000..1fe038e2 --- /dev/null +++ b/src/x402-live-quote.js @@ -0,0 +1,158 @@ +// Learn a seller's price from the only surface guaranteed to have it: a live 402. +// +// WHY THIS EXISTS (2026-08-07, reported by a seller who had just listed). +// Canu Verify listed 39 endpoints and every row came back price:null, +// priceUsd:0, payable:"unknown" - while each endpoint returns a textbook x402 v2 +// challenge (eip155:8453, Base USDC, amount 990000 = $0.99, real payTo) the +// moment you POST `{}` at it. Measured across the index the same day: 146 of +// 500 sellers had ZERO priced rows and 127 rows read payable:"unknown". So this +// was never one seller's problem. +// +// Two causes, both ours: +// 1. A manifest may list `resources` as bare URL STRINGS (theirs does, and +// the spec permits it). normaliseManifestTools can only read a price from +// an OBJECT, so a string-listing seller is permanently priceless no matter +// how well their endpoints behave. +// 2. probePaywall - the one thing that talks to a seller's endpoint - filters +// on `Number(t.price) > 0`. A route with no price is never probed, and +// probing is the only thing that would give it one. Circular by +// construction: the sellers who most need the probe are the only ones +// excluded from it. +// +// OpenAPI cannot close this: it has no place for an x402 quote. The 402 itself +// is the source of truth, which is also why the router already reads a live 402 +// for payTo (payToFromLive402) before spending. This reads the same challenge +// for price and networks. +// +// Deliberately CONSERVATIVE about money: an amount we cannot price leaves the +// price null and still records the networks, so the row becomes "payable over +// x402 on Base" rather than a guessed dollar figure. Under-claiming is the +// safe direction when the number decides what a buyer is charged. + +/** USDC is 6 decimals on every chain we accept. Anything else we refuse to + * price rather than guess - a wrong exponent is a 1000x pricing error. */ +const USDC_DECIMALS = 6; +const USDC_NAME = /^(usdc|usd coin)$/i; + +/** + * Pull the accepts array out of a live 402. + * + * x402 v2 carries it base64 in the `payment-required` HEADER with an empty (or + * unrelated) body; other sellers put it in the JSON body; Canu does BOTH, with + * the body nesting it under `payment`. All three are read, header first, + * because the header is the spec's home for it. + */ +export function acceptsFromLive402({ header, body } = {}) { + const dig = (obj) => { + if (!obj || typeof obj !== "object") return null; + if (Array.isArray(obj.accepts) && obj.accepts.length) return obj.accepts; + // Sellers wrap the envelope: { payment: { accepts } }, { x402: { accepts } }. + for (const k of ["payment", "x402", "paymentRequired", "payment_required", "data"]) { + const nested = obj[k]; + if (nested && typeof nested === "object" && Array.isArray(nested.accepts) && nested.accepts.length) { + return nested.accepts; + } + } + return null; + }; + + if (typeof header === "string" && header.trim()) { + try { + const decoded = JSON.parse(Buffer.from(header.trim(), "base64").toString("utf8")); + const hit = dig(decoded); + if (hit) return hit; + } catch { /* fall through to the body */ } + } + if (typeof body === "string" && body.trim()) { + try { + const hit = dig(JSON.parse(body)); + if (hit) return hit; + } catch { /* unreadable */ } + } + if (body && typeof body === "object") { + const hit = dig(body); + if (hit) return hit; + } + return null; +} + +/** Is this accepts entry denominated in USDC? Checked by NAME (what x402 v2 + * puts in `extra`) rather than by address, so a new chain's USDC works without + * a table to forget to update. */ +function isUsdc(a) { + return USDC_NAME.test(String(a?.extra?.name || "").trim()); +} + +/** + * Turn a live 402's accepts into the fields the index stores. + * + * Mirrors bazaarItemToTool's preference order deliberately - prefer Base USDC, + * then any USDC, then the first entry - so a row learned from a live probe and + * a row learned from the Bazaar are directly comparable. Two price ladders for + * the same catalogue would be worse than none. + * + * Returns price:null (never 0, never a guess) when the amount cannot be priced, + * while still returning the networks, because "payable on Base, amount unknown" + * is both true and useful, and 0 would read as free. + */ +export function quoteFromAccepts(accepts) { + const list = Array.isArray(accepts) ? accepts.filter((a) => a && typeof a === "object") : []; + if (!list.length) return null; + + const preferred = + list.find((a) => a.network === "eip155:8453" && isUsdc(a)) || + list.find(isUsdc) || + list[0]; + + let price = null; + const decimals = Number.isInteger(a$(preferred?.extra?.decimals)) ? a$(preferred.extra.decimals) + : isUsdc(preferred) ? USDC_DECIMALS + : null; + if (decimals != null && preferred?.amount != null) { + const n = Number(preferred.amount); + // A negative or non-finite amount is corrupt, not free. + if (Number.isFinite(n) && n >= 0) price = n / 10 ** decimals; + } + + return { + price, + networks: [...new Set(list.map((a) => a.network).filter((n) => typeof n === "string" && n))], + payTo: typeof preferred?.payTo === "string" ? preferred.payTo : null, + asset: typeof preferred?.asset === "string" ? preferred.asset : null, + // Which entry priced it, so a surprising number can be traced to its source. + network: typeof preferred?.network === "string" ? preferred.network : null, + }; +} + +/** Number() that refuses strings-that-are-not-numbers, for the decimals hint. */ +function a$(v) { + if (typeof v === "number") return v; + if (typeof v === "string" && /^\d+$/.test(v.trim())) return Number(v.trim()); + return NaN; +} + +/** + * Which HTTP methods to try, in order, for a route whose price we do not know. + * + * A GET-only prober cannot see a POST-only seller: Canu's endpoints 404 on GET + * and 402 on POST, so the whole catalogue read as priceless. When the catalogue + * states a method we trust it and try only that; when the method was INFERRED + * (or absent) we try GET then POST, because a POST with `{}` to a GET endpoint + * is harmless and a GET to a POST endpoint is a 404 that costs one request. + * + * Never PUT/PATCH/DELETE: an unpaid probe must not be able to mutate anything, + * even by accident, on a stranger's server. + */ +export function probeMethodsFor(tool) { + const stated = String(tool?.method || "").toUpperCase(); + if (stated === "POST") return ["POST"]; + if (stated === "GET" && tool?.methodInferred !== true) return ["GET", "POST"]; + if (stated && stated !== "GET" && stated !== "POST") return []; + return ["GET", "POST"]; +} + +/** Is this response a usable x402 quote? 402 is the only healthy answer to an + * unpaid call; a 200 means the route is not paywalled at all. */ +export function isQuoteResponse(status) { + return status === 402; +}