diff --git a/apps/scan/src/app/(app)/(home)/integration-spec/_content/markdown.ts b/apps/scan/src/app/(app)/(home)/integration-spec/_content/markdown.ts index 5b0b96570..eb7b7887b 100644 --- a/apps/scan/src/app/(app)/(home)/integration-spec/_content/markdown.ts +++ b/apps/scan/src/app/(app)/(home)/integration-spec/_content/markdown.ts @@ -105,6 +105,33 @@ SIWX routes are identity-gated, requiring a wallet proof but no payment. Agents The scheme **must** be named \`siwx\`. Discovery resolves it by name. Routes with both \`x-payment-info\` and \`siwx\` security are classified as paid, not SIWX. +## Free (Unprotected) Endpoints + +If your OpenAPI spec includes endpoints that are neither x402-paid nor SIWX (e.g. health checks, public read endpoints, webhooks), mark them with an empty \`security\` array: + +\`\`\`json +{ + "/v1/health": { + "get": { + "operationId": "health_check", + "security": [], + "summary": "Health check" + } + } +} +\`\`\` + +\`"security": []\` is the standard OpenAPI way to declare "no authentication required." Without it, the scanner can't distinguish free endpoints from paid ones that are misconfigured, and will probe them unnecessarily — producing errors and slowing registration. + +**Summary of endpoint classification:** + +| Type | OpenAPI Declaration | Scanner Behavior | +|---|---|---| +| Paid (x402) | \`x-payment-info\` + \`responses.402\` | Probed and registered | +| Identity-gated (SIWX) | \`security: [{ "siwx": [] }]\` | Registered as Free (no probe) | +| Free / public | \`security: []\` | Skipped entirely | +| Unclassified (no declaration) | Nothing | Probed (may fail if not x402) | + ## Common Failure Reasons | Error | Likely Cause | Fix | @@ -112,4 +139,5 @@ The scheme **must** be named \`siwx\`. Discovery resolves it by name. Routes wit | Not Found | OpenAPI not found at \`{origin}/openapi.json\` | Add an OpenAPI document at \`{origin}/openapi.json\` | | Input/Output Schema Missing | Operation has no input or output schema | Add an input and output schema to the operation | | No Payment Modes Detected | No payment modes detected in the response | Add a valid payment mode to the response (x402) | +| No valid x402 response / No 402 challenge | Endpoint is free but not marked as such | Add \`"security": []\` to the operation in your OpenAPI spec | `; diff --git a/apps/scan/src/app/(app)/(home)/resources/register/_components/form.tsx b/apps/scan/src/app/(app)/(home)/resources/register/_components/form.tsx index 28da29d28..0dfbd2692 100644 --- a/apps/scan/src/app/(app)/(home)/resources/register/_components/form.tsx +++ b/apps/scan/src/app/(app)/(home)/resources/register/_components/form.tsx @@ -6,9 +6,8 @@ import { Check, ChevronDown, Loader2, - Plus, + Minus, CircleHelp, - Trash2, TriangleAlert, X, } from 'lucide-react'; @@ -19,8 +18,6 @@ import { CollapsibleContent, CollapsibleTrigger, } from '@/components/ui/collapsible'; -import { Input } from '@/components/ui/input'; -import { Label } from '@/components/ui/label'; import { Tooltip, TooltipContent, @@ -108,9 +105,6 @@ function getPrimaryProbeError( export const RegisterResourceForm = () => { const [url, setUrl] = useState(''); const [httpWarning, setHttpWarning] = useState(false); - const [headers, setHeaders] = useState<{ name: string; value: string }[]>([]); - const [manualUrls, setManualUrls] = useState([]); - const [manualListError, setManualListError] = useState(null); const [manualProgress, setManualProgress] = useState<{ current: number; total: number; @@ -137,6 +131,7 @@ export const RegisterResourceForm = () => { resetBulk, preview, isBatchTestLoading, + batchTestProgress, testedResources, failedResources, retryResource, @@ -150,34 +145,24 @@ export const RegisterResourceForm = () => { const normalizedUrl = useMemo(() => normalizeUrl(url.trim()), [url]); - const queueOrigin = useMemo( - () => (manualUrls.length > 0 ? safeGetOrigin(manualUrls[0] ?? '') : null), - [manualUrls] - ); - const hasDiscoveryResources = discoveryFound && actualDiscoveredResources.length > 0; - // Always show the discovered count. The server re-discovers independently - // and will process all resources, so showing only the batch-test-passing - // subset is misleading (the success message would show a higher number). + // After batch test completes, count passing paid resources + SIWX (free) endpoints. + // SIWX endpoints aren't probed, so they aren't in testedResources — add them separately. + // Before batch test, fall back to total discovered count. const batchTestComplete = testedResources.length > 0 || failedResources.length > 0; - const registrableResourceCount = actualDiscoveredResources.length; + const siwxCount = actualDiscoveredResources.filter( + url => authModeMap[url] === 'siwx' + ).length; + const registrableResourceCount = batchTestComplete + ? testedResources.length + siwxCount + : actualDiscoveredResources.length; const canUseManualMode = isValidUrl && !isOriginOnly; - const currentUrlAlreadyInManualList = manualUrls.includes(normalizedUrl); - const canAddCurrentUrl = - canUseManualMode && - !currentUrlAlreadyInManualList && - (!queueOrigin || queueOrigin === urlOrigin); - - const manualTargets = - manualUrls.length > 0 - ? manualUrls - : canUseManualMode - ? [normalizedUrl] - : []; + + const manualTargets = canUseManualMode ? [normalizedUrl] : []; const testedResourceByUrl = useMemo(() => { const map = new Map(); @@ -202,27 +187,10 @@ export const RegisterResourceForm = () => { const activeBulkResult = manualResult ?? bulkData ?? null; const activeSummaryOrigin = manualResult?.origin ?? urlOrigin; - const requestHeaders = useMemo(() => { - const entries = headers - .map(header => ({ - name: header.name.trim(), - value: header.value, - })) - .filter(header => header.name.length > 0); - - if (entries.length === 0) { - return undefined; - } - - return Object.fromEntries( - entries.map(header => [header.name, header.value]) - ); - }, [headers]); const resetStateForNewRun = () => { setManualResult(null); setManualProgress(null); - setManualListError(null); resetBulk(); }; @@ -230,40 +198,12 @@ export const RegisterResourceForm = () => { setUrl(nextUrl); setManualResult(null); setManualProgress(null); - setManualListError(null); resetBulk(); }; - const handleAddCurrentUrl = () => { - if (!canUseManualMode) { - return; - } - - if (queueOrigin && queueOrigin !== urlOrigin) { - setManualListError('All manual URLs must share the same origin.'); - return; - } - - if (!currentUrlAlreadyInManualList) { - setManualUrls(current => [...current, normalizedUrl]); - } - - setManualListError(null); - setManualResult(null); - setManualProgress(null); - }; - - const handleRemoveManualUrl = (targetUrl: string) => { - setManualUrls(current => current.filter(item => item !== targetUrl)); - setManualResult(null); - setManualProgress(null); - setManualListError(null); - }; - const handleRegisterDiscovered = () => { setManualResult(null); setManualProgress(null); - setManualListError(null); handleRegisterAll(); }; @@ -294,7 +234,6 @@ export const RegisterResourceForm = () => { try { const result = await registerMutation.mutateAsync({ url: targetUrl, - headers: requestHeaders, }); if (result.success) { @@ -346,12 +285,6 @@ export const RegisterResourceForm = () => { await runManualRegistration([normalizedUrl]); }; - // Show advanced only after discovery fails or user has manual URLs - const showAdvanced = - (isValidUrl && !isDiscoveryLoading && !hasDiscoveryResources) || - manualUrls.length > 0 || - headers.length > 0; - const isLoading = isRegisteringAll || isRegisteringManual; return ( @@ -382,9 +315,6 @@ export const RegisterResourceForm = () => { x402 requires HTTPS. We've upgraded your URL automatically.

)} - {manualListError && ( -

{manualListError}

- )} {/* Primary action */} @@ -408,7 +338,9 @@ export const RegisterResourceForm = () => { ) : isBatchTestLoading ? ( <> - {`Verifying ${actualDiscoveredResources.length} endpoints...`} + {batchTestProgress + ? `Checking ${batchTestProgress.checked}/${batchTestProgress.total} endpoints...` + : `Checking ${actualDiscoveredResources.length} endpoints...`} ) : batchTestComplete && failedResources.length > 0 && @@ -544,129 +476,6 @@ export const RegisterResourceForm = () => { ); })()} - {/* Advanced — only when relevant */} - {showAdvanced && ( - - - - - - {/* Headers */} -
-
- - -
- {headers.map((header, index) => ( -
- - setHeaders(current => - current.map((item, itemIndex) => - itemIndex === index - ? { ...item, name: event.target.value } - : item - ) - ) - } - /> - - setHeaders(current => - current.map((item, itemIndex) => - itemIndex === index - ? { ...item, value: event.target.value } - : item - ) - ) - } - /> - -
- ))} -
- - {/* Manual URLs */} - {!hasDiscoveryResources && ( -
-
- - -
- {manualUrls.length > 0 && ( -
    - {manualUrls.map(item => ( -
  • - - {item} - - -
    • - ))} -
- )} -
- )} - - - )} - {/* Errors — endpoints that won't be registered */} {(() => { if ( @@ -698,7 +507,7 @@ export const RegisterResourceForm = () => { {' '} {isV1Issue ? 'This endpoint returns an x402 v1 response. x402scan only supports v2 — update your paywall to return the v2 format.' - : 'They need to return a 402 payment challenge — ensure the x402 paywall runs before request validation, or mark the required parameters in your OpenAPI spec so we can probe automatically.'} + : 'They need to return a 402 payment challenge — ensure the x402 paywall runs before request validation, or mark the required parameters in your OpenAPI spec so we can probe automatically. If these endpoints are free (not x402-paid), add "security": [] to their OpenAPI definition to exclude them from probing.'}

{failedResources.map((failed, idx) => ( @@ -713,10 +522,6 @@ export const RegisterResourceForm = () => {
0 && testedResources.length === 0 - } - v1Migration={isV1Issue} failedResources={failedResources.map(r => ({ url: r.url, error: getPrimaryProbeError(r), @@ -728,6 +533,63 @@ export const RegisterResourceForm = () => { ); })()} + {/* Warnings — endpoints that will register but have issues */} + {(() => { + if (activeBulkResult || isBatchTestLoading) return null; + const resourcesWithWarnings = testedResources.filter( + r => r.warnings && r.warnings.length > 0 + ); + if (resourcesWithWarnings.length === 0) return null; + + return ( + + + + + +

+ These endpoints will still be registered, but have issues that + may affect agent compatibility. +

+
+ {resourcesWithWarnings.map((r, idx) => ( +
+
+ {toPathLabel(r.url)} +
+ {r.warnings?.map((w, wi) => ( +
+ {w.message} +
+ ))} +
+ ))} +
+ + (r.warnings ?? []).map(w => ({ + url: r.url, + error: w.message, + })) + )} + /> + + + ); + })()} + {/* Bulk result */} {activeBulkResult && activeSummaryOrigin ? ( ; @@ -836,6 +698,15 @@ function ProbeResult({ () => new Set(testedResources.map(r => r.url)), [testedResources] ); + const warningUrls = useMemo( + () => + new Set( + testedResources + .filter(r => r.warnings && r.warnings.length > 0) + .map(r => r.url) + ), + [testedResources] + ); const failedUrls = useMemo( () => new Set(failedResources.map(r => r.url)), [failedResources] @@ -849,6 +720,17 @@ function ProbeResult({ ), [authModeMap] ); + const nonPaidUrls = useMemo(() => { + const paid = new Set(['paid', 'apiKey+paid']); + return new Set( + resources.filter(url => { + const mode = authModeMap[url]; + // Only mark as non-paid if discovery explicitly classified it + // as non-paid. If authMode is missing, don't pre-judge. + return mode !== undefined && mode !== 'siwx' && !paid.has(mode); + }) + ); + }, [resources, authModeMap]); const invalidUrls = useMemo( () => new Set( @@ -858,13 +740,17 @@ function ProbeResult({ ), [invalidResourcesMap] ); + // Sort: errors → warnings → free (SIWX) → verified → skipped const sortedResources = useMemo(() => { const priority = (url: string) => { if (invalidUrls.has(url) || failedUrls.has(url)) return 0; - return 2; + if (warningUrls.has(url)) return 1; + if (siwxUrls.has(url)) return 2; + if (testedUrls.has(url)) return 3; + return 4; // non-paid — skipped }; return [...resources].sort((a, b) => priority(a) - priority(b)); - }, [resources, invalidUrls, failedUrls]); + }, [resources, invalidUrls, failedUrls, warningUrls, siwxUrls, testedUrls]); const [expanded, setExpanded] = useState(false); const previewResources = expanded @@ -935,16 +821,28 @@ function ProbeResult({ key={resource} className="font-mono truncate flex items-center gap-1.5" > - {invalidUrls.has(resource) ? ( + {nonPaidUrls.has(resource) ? ( + + ) : invalidUrls.has(resource) ? ( ) : siwxUrls.has(resource) ? ( - + + ) : warningUrls.has(resource) ? ( + ) : testedUrls.has(resource) ? ( ) : failedUrls.has(resource) ? ( ) : null} - {toPathLabel(resource)} + + {toPathLabel(resource)} + ))} {!expanded && hiddenCount > 0 && ( diff --git a/apps/scan/src/app/(app)/_components/discovery/discovery-actions.tsx b/apps/scan/src/app/(app)/_components/discovery/discovery-actions.tsx index acba96da7..df05365b4 100644 --- a/apps/scan/src/app/(app)/_components/discovery/discovery-actions.tsx +++ b/apps/scan/src/app/(app)/_components/discovery/discovery-actions.tsx @@ -6,131 +6,102 @@ import { toast } from 'sonner'; import { Button } from '@/components/ui/button'; import { useCopyToClipboard } from '@/hooks/use-copy-to-clipboard'; -// Duplicated from discovery/_constants/prompts.ts to avoid cross-directory -// import issues with Turbopack in dev. Keep in sync. -const GENERIC_PROMPT = - "Read https://agentcash.dev/merchants.md and follow the guide to make my API discoverable and payable by agents. Do everything automatically. Only ask me if you need input you can't determine yourself."; - -function buildErrorPrompt( - resources: { url: string; error: string; status?: number }[], - missingSchemaUrls?: string[] -): string { - const lines = resources.map(r => { - const status = r.status ? ` (HTTP ${r.status})` : ''; - return `- ${r.url}: ${r.error}${status}`; - }); - - let schemaSection = ''; - if (missingSchemaUrls && missingSchemaUrls.length > 0) { - const schemaLines = missingSchemaUrls.map(u => `- ${u}`); - schemaSection = ` - -These endpoints are also missing input schemas (agents won't know what request to send): - -${schemaLines.join('\n')} - -For each, add a requestBody schema and/or parameter definitions to the OpenAPI spec.`; - } - - return `These x402 endpoints failed registration on x402scan.com: - -${lines.join('\n')}${schemaSection} +const SETUP_PROMPT = `My API doesn't have a discovery document yet. Create an OpenAPI spec (openapi.json) that describes my endpoints, then serve it so x402scan.com can discover them. Read https://x402scan.com/discovery/spec for the full discovery specification. -For each failing endpoint: -1. Verify the endpoint returns a 402 status with valid x402 payment headers when called without payment -2. Check that request validation (body schema, query params) doesn't reject the request before the x402 middleware runs -3. In the OpenAPI spec, mark all required query parameters with "required": true — x402scan probes endpoints automatically and needs to know which parameters are required to test successfully -4. Ensure the endpoint is listed in your OpenAPI spec with the correct method and input schema - -Fix each issue. Only ask me if you need input you can't determine yourself.`; -} - -function buildWarningPrompt( - resources: { url: string; error: string; status?: number }[] -): string { - const lines = resources.map(r => { - const status = r.status ? ` (HTTP ${r.status})` : ''; - return `- ${r.url}: ${r.error}${status}`; - }); - return `These endpoints were found in my API spec but triggered warnings during registration on x402scan.com: - -${lines.join('\n')} - -If these endpoints are meant to be paid, read https://x402scan.com/discovery/spec and ensure each one returns a 402 status with valid x402 payment headers when called without payment. +Steps: +1. Identify all endpoints in my API — both paid (x402) and free (identity-gated) +2. Create an openapi.json with paths, methods, request/response schemas, and descriptions +3. Serve it at /openapi.json (or /.well-known/x402 pointing to it) +4. Paid endpoints must return a 402 with valid x402 v2 payment headers when called without payment +5. Free endpoints should declare \`"security": []\` in the OpenAPI spec -If they're intentionally free, remove them from the x402 discovery document so they don't show as warnings. +Do everything automatically. Only ask me if you need input you can't determine yourself.`; -Only ask me if you need input you can't determine yourself.`; -} +function buildConsolidatedPrompt({ + failedResources, + warnings, + missingSchemaResources, +}: { + failedResources?: { url: string; error: string; status?: number }[]; + warnings?: { url: string; error: string; status?: number }[]; + missingSchemaResources?: string[]; +}): string { + const sections: string[] = []; -const V1_MIGRATION_PROMPT = `My x402 endpoints are returning v1 payment responses. Migrate them to the x402 v2 spec. + if (failedResources && failedResources.length > 0) { + const lines = failedResources.map(r => { + const status = r.status ? ` (HTTP ${r.status})` : ''; + return `- ${r.url}: ${r.error}${status}`; + }); + sections.push(`Errors (these endpoints failed and won't be registered): -Key changes from v1 to v2: -- The 402 response body must include \`"x402Version": 2\` -- The \`accepts\` array replaces \`maxAmountRequired\` with \`amount\` (the exact price for this request) -- Each entry in \`accepts\` must include: scheme, network, asset, amount, payTo -- Remove \`maxTimeoutSeconds\` from accepts entries (moved to top-level \`x402Version\` sibling if needed) +${lines.join('\n')}`); + } -Read https://x402scan.com/discovery/spec for the full v2 specification. + if (warnings && warnings.length > 0) { + const lines = warnings.map(r => { + const status = r.status ? ` (HTTP ${r.status})` : ''; + return `- ${r.url}: ${r.error}${status}`; + }); + sections.push(`Warnings (registered but with issues): -Update every endpoint that returns a v1 response. Only ask me if you need input you can't determine yourself.`; +${lines.join('\n')}`); + } -const CREATE_OPENAPI_PROMPT = `My API doesn't have a discovery document yet. Create an OpenAPI spec (openapi.json) that describes my x402-paid endpoints, then serve it so x402scan.com can discover them. + if (missingSchemaResources && missingSchemaResources.length > 0) { + const lines = missingSchemaResources.map(u => `- ${u}`); + sections.push(`Missing input schemas (agents won't know what request to send): -Read https://x402scan.com/discovery/spec for the full discovery specification. +${lines.join('\n')}`); + } -Steps: -1. Identify all paid endpoints in my API -2. Create an openapi.json with paths, methods, request/response schemas, and descriptions -3. Serve it at /openapi.json (or /.well-known/x402 pointing to it) -4. Ensure each paid endpoint returns a 402 with valid x402 v2 payment headers when called without payment + // Shouldn't be reachable (prompt only shown when there are issues), but + // fall back to the spec link rather than a misleading generic message. + if (sections.length === 0) { + return 'Read https://x402scan.com/discovery/spec for the full discovery specification. Follow the guide to ensure your endpoints are correctly configured for x402scan.'; + } -Do everything automatically. Only ask me if you need input you can't determine yourself.`; + const issueBlock = sections.join('\n\n'); -const ADD_SCHEMA_PROMPT = `Some of my x402 endpoints are missing input/output schemas in the OpenAPI spec. Without these, agents can find and pay for the endpoint but don't know what request to send. + return `${issueBlock} Read https://x402scan.com/discovery/spec for the full discovery specification. -For each endpoint in my OpenAPI spec: -1. Add a requestBody schema describing the expected JSON body (field names, types, descriptions, required fields) -2. Add response schemas describing what the endpoint returns -3. Add parameter definitions for any required query parameters +To fix these: +1. Paid endpoints must return a 402 status with valid x402 v2 payment headers when called without payment +2. Request validation (body schema, query params) must not reject the request before the x402 middleware runs +3. Mark all required query parameters with "required": true in the OpenAPI spec — x402scan probes endpoints automatically +4. Add request/response schemas to the OpenAPI spec so agents know what to send and expect back +5. Free (identity-gated) endpoints should declare \`"security": []\` in the OpenAPI spec -Do everything automatically. Only ask me if you need input you can't determine yourself.`; +Fix each issue. Only ask me if you need input you can't determine yourself.`; +} export function DiscoveryActions({ iconOnly, label, failedResources, warnings, - v1Migration, noDiscovery, - missingSchema, missingSchemaResources, }: { iconOnly?: boolean; label?: string; failedResources?: { url: string; error: string; status?: number }[]; warnings?: { url: string; error: string; status?: number }[]; - v1Migration?: boolean; noDiscovery?: boolean; - missingSchema?: boolean; - /** URLs missing input schemas — merged into the error prompt when both exist. */ + /** URLs missing input schemas — merged into the consolidated prompt. */ missingSchemaResources?: string[]; }) { const prompt = noDiscovery - ? CREATE_OPENAPI_PROMPT - : v1Migration - ? V1_MIGRATION_PROMPT - : failedResources && failedResources.length > 0 - ? buildErrorPrompt(failedResources, missingSchemaResources) - : missingSchema - ? ADD_SCHEMA_PROMPT - : warnings && warnings.length > 0 - ? buildWarningPrompt(warnings) - : GENERIC_PROMPT; + ? SETUP_PROMPT + : buildConsolidatedPrompt({ + failedResources, + warnings, + missingSchemaResources, + }); const { isCopied, copyToClipboard } = useCopyToClipboard(() => { toast.success('Copied prompt for agents'); diff --git a/apps/scan/src/app/(app)/_components/discovery/discovery-panel.tsx b/apps/scan/src/app/(app)/_components/discovery/discovery-panel.tsx index 6b73dfac0..77066a55a 100644 --- a/apps/scan/src/app/(app)/_components/discovery/discovery-panel.tsx +++ b/apps/scan/src/app/(app)/_components/discovery/discovery-panel.tsx @@ -65,45 +65,26 @@ export function DiscoveryFixHint({ className, failedResources, warnings, - v1Migration, noDiscovery, - needsSetup, - missingSchema, missingSchemaResources, }: { className?: string; failedResources?: { url: string; error: string; status?: number }[]; warnings?: { url: string; error: string; status?: number }[]; - v1Migration?: boolean; noDiscovery?: boolean; - /** All endpoints failed — treat as needing x402 setup from scratch. */ - needsSetup?: boolean; - missingSchema?: boolean; missingSchemaResources?: string[]; }) { const label = noDiscovery ? 'Have your agent create an OpenAPI spec for your resource' - : needsSetup - ? 'Set up x402 with a prompt' - : v1Migration - ? 'Migrate to x402 v2 spec in one prompt' - : missingSchema - ? 'Add input schemas with a prompt' - : 'Have your agent fix the errors with a prompt'; + : 'Have your agent fix the issues with a prompt'; return (

{' '} or{' '} @@ -384,7 +365,7 @@ export function DiscoveryPanel({ {bulkResult.warningDetails.length} resource {bulkResult.warningDetails.length === 1 ? '' : 's'} registered - with warnings + with warnings (Not blocking)

@@ -899,7 +880,7 @@ function FailedResourceCard({ const isV1Error = failedDetails?.error?.includes('v1 response detected') ?? false; const errorMessage = isSiwx - ? 'SIWX (identity-gated, no payment)' + ? 'Free (wallet auth, no payment)' : isInvalid ? (invalidInfo?.reason ?? 'Invalid format') : x402Parsed @@ -913,7 +894,7 @@ function FailedResourceCard({ className={cn( 'overflow-hidden', isSiwx - ? 'border-primary' + ? 'border-green-600' : isInvalid ? 'border-yellow-500/30' : 'border-red-500/30' @@ -931,13 +912,13 @@ function FailedResourceCard({ className={cn( 'font-mono px-1 rounded-md text-xs shrink-0', isSiwx - ? 'bg-primary/10 border border-primary text-primary' + ? 'bg-green-600/10 border border-green-600 text-green-600' : isInvalid ? 'bg-yellow-600/10 border border-yellow-600 text-yellow-600' : 'bg-red-600/10 border border-red-600 text-red-600' )} > - {isSiwx ? 'SIWX' : isInvalid ? 'INVALID' : 'ERR'} + {isSiwx ? 'FREE' : isInvalid ? 'INVALID' : 'ERR'}
{pathname}
@@ -979,19 +960,15 @@ function FailedResourceCard({ ]} /> - {isV1Error ? ( - - ) : ( - - )} + {/* Sample body input — shown for reachable endpoints that didn't return 402 */} {onRetry && !isSiwx && !isV1Error && ( @@ -1179,6 +1156,17 @@ function RegisterModeResourceList({ if (allResources.length === 0) return null; + // Sort: invalid → free (SIWX) → new → already registered + const sortedResources = [...allResources].sort((a, b) => { + const priority = (r: (typeof allResources)[number]) => { + if (invalidResourcesMap[r.url]?.invalid) return 0; + if (authModeMap[r.url] === 'siwx') return 1; + if (!r.isRegistered) return 2; + return 3; // already registered + }; + return priority(a) - priority(b); + }); + return (
@@ -1190,87 +1178,91 @@ function RegisterModeResourceList({ - {allResources.map(({ url, source: resourceSource, isRegistered }) => { - const pathname = (() => { - try { - return decodeURIComponent(new URL(url).pathname); - } catch { - return url; - } - })(); - - return ( - - - - - - ); - })} + {sortedResources.map( + ({ url, source: resourceSource, isRegistered }) => { + const pathname = (() => { + try { + return decodeURIComponent(new URL(url).pathname); + } catch { + return url; + } + })(); + + return ( + + + + + + ); + } + )}
- {pathname} - - - {resourceSource === 'entered' - ? 'Manually Entered' - : resourceSource === 'openapi' - ? 'OpenAPI' - : resourceSource === 'probe' - ? 'Runtime Probe' - : resourceSource === 'interop-mpp' - ? '/.well-known/mpp' - : '/.well-known/x402'} - - -
- {isRegistered ? ( - - - Already Registered - - ) : ( - New - )} - {authModeMap[url] === 'siwx' && ( - - - - - SIWX - - - -

- Identity-gated route (Sign-In With X). Requires a - wallet proof; no payment. -

- - - )} - {invalidResourcesMap[url]?.invalid && ( - - - - - INVALID - - - -

- {invalidResourcesMap[url]?.reason ?? - 'Invalid format'} -

- - - )} -
-
+ {pathname} + + + {resourceSource === 'entered' + ? 'Manually Entered' + : resourceSource === 'openapi' + ? 'OpenAPI' + : resourceSource === 'probe' + ? 'Runtime Probe' + : resourceSource === 'interop-mpp' + ? '/.well-known/mpp' + : '/.well-known/x402'} + + +
+ {isRegistered ? ( + + + Already Registered + + ) : ( + + New + + )} + {authModeMap[url] === 'siwx' && ( + + + + + Free + + + +

+ Free endpoint — requires wallet authentication but + no payment. +

+ + + )} + {invalidResourcesMap[url]?.invalid && ( + + + + + INVALID + + + +

+ {invalidResourcesMap[url]?.reason ?? + 'Invalid format'} +

+ + + )} +
+
diff --git a/apps/scan/src/app/(app)/_components/discovery/use-batch-test.ts b/apps/scan/src/app/(app)/_components/discovery/use-batch-test.ts index fe4f26845..4ae9b2c58 100644 --- a/apps/scan/src/app/(app)/_components/discovery/use-batch-test.ts +++ b/apps/scan/src/app/(app)/_components/discovery/use-batch-test.ts @@ -5,11 +5,20 @@ import { api } from '@/trpc/client'; import type { TestedResource, FailedResource } from '@/types/batch-test'; import type { DiscoveredResource } from '@/types/discovery'; +export interface BatchTestProgress { + checked: number; + total: number; +} + interface BatchTestResult { isLoading: boolean; + progress: BatchTestProgress | null; resources: TestedResource[]; failed: FailedResource[]; payToAddresses: string[]; + /** Server-side probe session ID. Pass to registerFromOrigin so the server + * reuses cached probe results instead of re-probing. */ + probeSessionId: string | null; refetch: () => void; retryOne: ( url: string, @@ -17,7 +26,10 @@ interface BatchTestResult { ) => Promise; } -const BATCH_SIZE = 20; +// One endpoint per request for per-endpoint progress updates. +// The server probes sequentially anyway, so N requests of 1 endpoint +// has the same total probe time as 1 request of N endpoints. +const BATCH_SIZE = 1; /** * Split array into chunks of specified size @@ -43,7 +55,9 @@ export function useBatchTest( const [resources, setResources] = useState([]); const [failed, setFailed] = useState([]); const [isLoading, setIsLoading] = useState(false); + const [progress, setProgress] = useState(null); const [runCount, setRunCount] = useState(0); + const [probeSessionId, setProbeSessionId] = useState(null); const mutation = api.developer.batchTest.useMutation(); const mutateAsyncRef = useRef(mutation.mutateAsync); @@ -51,9 +65,21 @@ export function useBatchTest( mutateAsyncRef.current = mutation.mutateAsync; }); + // Sort paid endpoints first so they're probed before unclassified ones. + // This prevents rate limiting from burning through all probes on non-paid + // endpoints before reaching the ones that actually matter. + const sortedResources = useMemo(() => { + const paidModes = new Set(['paid', 'apiKey+paid']); + return [...effectiveResources].sort((a, b) => { + const aPaid = a.authMode != null && paidModes.has(a.authMode) ? 0 : 1; + const bPaid = b.authMode != null && paidModes.has(b.authMode) ? 0 : 1; + return aPaid - bPaid; + }); + }, [effectiveResources]); + const chunks = useMemo( - () => chunkArray(effectiveResources, BATCH_SIZE), - [effectiveResources] + () => chunkArray(sortedResources, BATCH_SIZE), + [sortedResources] ); useEffect(() => { @@ -61,27 +87,41 @@ export function useBatchTest( let cancelled = false; - const request = Promise.all( - chunks.map(chunk => mutateAsyncRef.current({ resources: chunk })) - ); + const run = async () => { + if (!cancelled) { + setIsLoading(true); + setProbeSessionId(null); + setProgress({ checked: 0, total: effectiveResources.length }); + } - void Promise.resolve() - .then(() => { - if (!cancelled) setIsLoading(true); - }) - .then(() => request) - .then(results => { - if (cancelled) return; - const allResources: TestedResource[] = []; - const allFailed: FailedResource[] = []; - for (const result of results) { + const allResources: TestedResource[] = []; + const allFailed: FailedResource[] = []; + let sessionId: string | undefined; + + try { + for (const chunk of chunks) { + if (cancelled) return; + const result = await mutateAsyncRef.current({ + resources: chunk, + // Pass sessionId from first chunk so all results share one cache + ...(sessionId ? { probeSessionId: sessionId } : {}), + }); + sessionId ??= result.probeSessionId; + if (!cancelled && sessionId) { + setProbeSessionId(sessionId); + } allResources.push(...result.resources); allFailed.push(...result.failed); + if (!cancelled) { + setResources([...allResources]); + setFailed([...allFailed]); + setProgress({ + checked: allResources.length + allFailed.length, + total: effectiveResources.length, + }); + } } - setResources(allResources); - setFailed(allFailed); - }) - .catch(err => { + } catch (err) { if (cancelled) return; const error = err instanceof Error ? err.message : 'Request failed'; setFailed( @@ -89,15 +129,20 @@ export function useBatchTest( .flat() .map(r => ({ success: false as const, url: r.url, error })) ); - }) - .finally(() => { - if (!cancelled) setIsLoading(false); - }); + } finally { + if (!cancelled) { + setIsLoading(false); + setProgress(null); + } + } + }; + + void run(); return () => { cancelled = true; }; - }, [enabled, chunks, runCount]); + }, [enabled, chunks, runCount, effectiveResources.length]); const active = enabled && chunks.length > 0; @@ -132,6 +177,8 @@ export function useBatchTest( sampleBody: options?.sampleBody, }, ], + // Append to existing session so retried probes land in the same cache + ...(probeSessionId ? { probeSessionId } : {}), }); // Merge result: replace existing entry for the original URL @@ -154,9 +201,11 @@ export function useBatchTest( return { isLoading: isLoading && active, + progress: active ? progress : null, resources: active ? resources : [], failed: active ? failed : [], payToAddresses, + probeSessionId: active ? probeSessionId : null, refetch: () => setRunCount(c => c + 1), retryOne, }; diff --git a/apps/scan/src/app/(app)/_components/discovery/use-discovery.ts b/apps/scan/src/app/(app)/_components/discovery/use-discovery.ts index efe778233..9840d2dee 100644 --- a/apps/scan/src/app/(app)/_components/discovery/use-discovery.ts +++ b/apps/scan/src/app/(app)/_components/discovery/use-discovery.ts @@ -78,6 +78,7 @@ export interface UseDiscoveryReturn { // Test results isBatchTestLoading: boolean; + batchTestProgress: { checked: number; total: number } | null; testedResources: TestedResource[]; failedResources: FailedResource[]; @@ -127,6 +128,9 @@ export interface UseDiscoveryReturn { ) => Promise; } +/** Auth modes that are relevant to x402scan (paid + free/SIWX). */ +const REGISTRABLE_AUTH_MODES = new Set(['paid', 'apiKey+paid', 'siwx']); + export function useDiscovery({ url, onRegisterAllSuccess, @@ -157,10 +161,16 @@ export function useDiscovery({ ); const discoveryFound = discoveryQuery.data?.found ?? false; - const discoveryResources: DiscoveredResource[] = useMemo( - () => (discoveryQuery.data?.found ? discoveryQuery.data.resources : []), - [discoveryQuery.data] - ); + const discoveryResources: DiscoveredResource[] = useMemo(() => { + const raw = discoveryQuery.data?.found ? discoveryQuery.data.resources : []; + // Exclude endpoints explicitly classified as non-registrable (unprotected, + // apiKey). Keep registrable (paid, siwx) and unclassified (authMode absent) + // — unclassified endpoints may be paid but the discovery package didn't + // detect it (e.g. bazaar-only endpoints). + return raw.filter( + r => r.authMode == null || REGISTRABLE_AUTH_MODES.has(r.authMode) + ); + }, [discoveryQuery.data]); const discoveryCheckComplete = !discoveryQuery.isLoading && discoveryQuery.isFetched; @@ -288,11 +298,13 @@ export function useDiscovery({ onError: () => onRegisterAllError?.(), }); - // Handle registering all discovered resources + // Handle registering all discovered resources. + // Pass the probe session ID so the server reuses cached probe results + // without advisory data ever round-tripping through the client. const handleRegisterAll = () => { if (!urlOrigin) return; resetBulk(); - void register(urlOrigin); + void register(urlOrigin, batchTest.probeSessionId ?? undefined); }; return { @@ -325,6 +337,7 @@ export function useDiscovery({ // Test results isBatchTestLoading: batchTest.isLoading, + batchTestProgress: batchTest.progress, testedResources: batchTest.resources, failedResources: batchTest.failed, diff --git a/apps/scan/src/hooks/use-register-from-origin.ts b/apps/scan/src/hooks/use-register-from-origin.ts index 1e85f2aea..064b1108f 100644 --- a/apps/scan/src/hooks/use-register-from-origin.ts +++ b/apps/scan/src/hooks/use-register-from-origin.ts @@ -88,7 +88,8 @@ export function useRegisterFromOrigin( }, }); - const register = (origin: string) => mutation.mutateAsync({ origin }); + const register = (origin: string, probeSessionId?: string) => + mutation.mutateAsync({ origin, probeSessionId }); const error = mutation.data && !mutation.data.success diff --git a/apps/scan/src/lib/discovery/probe-cache.ts b/apps/scan/src/lib/discovery/probe-cache.ts new file mode 100644 index 000000000..ff14ec228 --- /dev/null +++ b/apps/scan/src/lib/discovery/probe-cache.ts @@ -0,0 +1,77 @@ +import { randomUUID } from 'crypto'; +import type { + AuditWarning, + EndpointMethodAdvisory, +} from '@agentcash/discovery'; +import { getRedisClient } from '@/lib/redis'; + +/** + * Server-side probe session cache. + * + * SECURITY: Probe results (advisories) contain merchant-controlled data such as + * payTo addresses. They must never round-trip through the client — a malicious + * actor could tamper with them to redirect payments. This cache keeps probe + * results server-side: the client only receives an opaque sessionId and passes + * it back at registration time. + */ + +const PROBE_SESSION_TTL_SECONDS = 5 * 60; // 5 minutes +const KEY_PREFIX = 'probe-session'; + +interface CachedProbeResult { + advisory: EndpointMethodAdvisory; + warnings: AuditWarning[]; +} + +function cacheKey(sessionId: string, url: string): string { + return `${KEY_PREFIX}:${sessionId}:${url}`; +} + +export function createProbeSession(): string { + return randomUUID(); +} + +export async function cacheProbeResult( + sessionId: string, + url: string, + advisory: EndpointMethodAdvisory, + warnings: AuditWarning[] +): Promise { + const redis = getRedisClient(); + if (!redis) return; + + try { + await redis.setex( + cacheKey(sessionId, url), + PROBE_SESSION_TTL_SECONDS, + JSON.stringify({ advisory, warnings } satisfies CachedProbeResult) + ); + } catch (err) { + console.error('[probe-cache] Failed to cache probe result:', err); + } +} + +export async function getCachedProbeResult( + sessionId: string, + url: string +): Promise { + const redis = getRedisClient(); + if (!redis) return null; + + try { + const raw = await redis.get(cacheKey(sessionId, url)); + if (!raw) return null; + const parsed: unknown = JSON.parse(raw); + if ( + typeof parsed !== 'object' || + parsed === null || + !('advisory' in parsed) + ) { + return null; + } + return parsed as CachedProbeResult; + } catch (err) { + console.error('[probe-cache] Failed to read cached probe result:', err); + return null; + } +} diff --git a/apps/scan/src/lib/discovery/probe.ts b/apps/scan/src/lib/discovery/probe.ts index 24cbfe39c..4b4e3b2a9 100644 --- a/apps/scan/src/lib/discovery/probe.ts +++ b/apps/scan/src/lib/discovery/probe.ts @@ -1,15 +1,65 @@ -import { checkEndpointSchema, getWarningsForL3 } from '@agentcash/discovery'; +import { + checkEndpointSchema, + getWarningsForL3, + validatePaymentRequiredDetailed, +} from '@agentcash/discovery'; import type { AuditWarning, CheckEndpointResult, EndpointMethodAdvisory, } from '@agentcash/discovery'; +import https from 'node:https'; import { buildMinimalSampleFromInputSchema, buildMinimalQueryParamsFromInputSchema, PROBE_TIMEOUT_MS, } from './utils'; +/** + * Direct HTTPS probe that tolerates large response headers (128 KB). + * Some merchants embed the full 402 payment body in a `payment-required` + * header, exceeding Node's default 16 KB limit. The discovery library's + * internal `fetch` silently drops these as network errors. + */ +function directProbe402( + url: string, + method: string, + timeoutMs: number +): Promise<{ status: number; body: unknown } | null> { + return new Promise(resolve => { + const parsed = new URL(url); + const req = https.request( + { + hostname: parsed.hostname, + port: parsed.port || 443, + path: parsed.pathname + parsed.search, + method, + headers: { Accept: 'application/json' }, + timeout: timeoutMs, + maxHeaderSize: 128 * 1024, + }, + res => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + try { + const body: unknown = JSON.parse(Buffer.concat(chunks).toString()); + resolve({ status: res.statusCode ?? 0, body }); + } catch { + resolve(null); + } + }); + } + ); + req.on('error', () => resolve(null)); + req.on('timeout', () => { + req.destroy(); + resolve(null); + }); + req.end(); + }); +} + export type ProbeX402Result = | { success: true; @@ -76,8 +126,18 @@ function pickInputSchemaFromSpec( return advisory?.inputSchema; } +const RATE_LIMIT_STATUSES = new Set([429, 503]); +const MAX_RETRIES = 2; +const RETRY_BASE_MS = 1500; + +/** Sleep for a given number of milliseconds. */ +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + /** * Probes a URL and returns the first advisory that carries x402 payment options. + * Retries up to MAX_RETRIES times on 429/503 with exponential backoff. * * Strategy: * 1. Probe with no body (works for servers whose paywall fires before @@ -96,6 +156,36 @@ export async function probeX402Endpoint( url: string, preferredMethod?: string, sampleBody?: Record +): Promise { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + const result = await probeX402EndpointOnce( + url, + preferredMethod, + sampleBody + ); + + // Retry on rate limiting (429/503) with exponential backoff. + if ( + !result.success && + result.statusCode !== undefined && + RATE_LIMIT_STATUSES.has(result.statusCode) && + attempt < MAX_RETRIES + ) { + await sleep(RETRY_BASE_MS * 2 ** attempt); + continue; + } + + return result; + } + + // Unreachable, but satisfies the type checker. + return { success: false, error: 'Max retries exceeded' }; +} + +async function probeX402EndpointOnce( + url: string, + preferredMethod?: string, + sampleBody?: Record ): Promise { const noBody = await checkEndpointSchema({ url, @@ -153,6 +243,71 @@ export async function probeX402Endpoint( }; } + // Fallback: direct HTTPS probe with large-header tolerance. + // Some merchants embed the full 402 body in a `payment-required` header, + // exceeding Node's default 16 KB limit. The discovery library's probes + // fail silently on these endpoints. + const directMethod = preferredMethod?.toUpperCase() ?? 'GET'; + const direct = await directProbe402(probeUrl, directMethod, PROBE_TIMEOUT_MS); + if (direct?.status === 402 && direct.body) { + // Validate the 402 body and extract normalized accepts. + const validated = validatePaymentRequiredDetailed(direct.body); + if (validated.valid && validated.normalized) { + // Build advisory from OpenAPI (schemas) + direct 402 body (payment options). + // The library's probe failed due to header overflow, but OpenAPI may + // still have schema info we can merge in. + const retry = await checkEndpointSchema({ + url: probeUrl, + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + const openApiAdvisory = pickX402Advisory(retry, preferredMethod); + + // Extract payment options from the validated 402 body's accepts array. + // These are already in the x402 wire format — the library normally + // parses them from the probe response, but header overflow prevented it. + // `validated.normalized` confirms the body parsed as valid x402, so the + // accepts array is structurally sound. + const rawBody = direct.body as Record; + const rawAccepts = Array.isArray(rawBody.accepts) ? rawBody.accepts : []; + const paymentOptions = rawAccepts + .filter( + (a): a is Record => + typeof a === 'object' && a !== null && 'payTo' in a + ) + .map(accept => ({ + protocol: 'x402' as const, + ...accept, + })) as NonNullable; + + const advisory: EndpointMethodAdvisory = { + source: 'probe', + method: directMethod as EndpointMethodAdvisory['method'], + paymentOptions, + paymentRequiredBody: direct.body, + ...(openApiAdvisory?.inputSchema + ? { inputSchema: openApiAdvisory.inputSchema } + : {}), + ...(openApiAdvisory?.outputSchema + ? { outputSchema: openApiAdvisory.outputSchema } + : {}), + ...(openApiAdvisory?.summary + ? { summary: openApiAdvisory.summary } + : {}), + }; + + const warnings = getWarningsForL3(advisory); + warnings.push({ + code: 'HEADERS_OVERFLOW', + severity: 'warn', + message: + 'Response headers exceed 16 KB — the payment-required header may be too large. ' + + 'Some clients will fail to parse this response. ' + + 'Consider moving the full payment body to the 402 JSON response only.', + }); + return { success: true, advisory, warnings }; + } + } + const isUnreachable = !noBody.found && (noBody.cause === 'network' || noBody.cause === 'timeout'); diff --git a/apps/scan/src/lib/discovery/register-origin.ts b/apps/scan/src/lib/discovery/register-origin.ts index 0c6062c26..a38945886 100644 --- a/apps/scan/src/lib/discovery/register-origin.ts +++ b/apps/scan/src/lib/discovery/register-origin.ts @@ -1,12 +1,17 @@ import { probeX402Endpoint } from './probe'; +import { getCachedProbeResult } from './probe-cache'; import { getRegistrationErrorMessage } from './utils'; -import { registerResource } from '@/lib/resources'; +import { registerResource, registerSiwxResource } from '@/lib/resources'; import { deprecateStaleResources } from '@/services/db/resources/resource'; import { getOriginResourceCount } from '@/services/db/resources/origin'; import { notifyNewServer } from '@/lib/discord-notifications'; import { getOriginFromUrl } from '@/lib/url'; -import type { AuthMode } from '@agentcash/discovery'; +import type { + AuditWarning, + AuthMode, + EndpointMethodAdvisory, +} from '@agentcash/discovery'; const BULK_REGISTER_CONCURRENCY = 6; @@ -73,10 +78,9 @@ export interface RegisterOriginResult { /** * Probe and register all resources from a discovery document. * Paid resources are probed and written to the resources table. - * SIWX-identified routes are a first-class positive outcome — they are - * reported back in `siwx`/`siwxDetails` but not written to the DB (until - * schema support lands). Endpoints missing an input schema are reported - * as skipped. + * SIWX (free) resources are written to the resources table without probing + * (they have no x402 payment options — just a Resource row, no Accepts). + * Endpoints missing an input schema are reported as skipped. * Deprecates resources from the same origin that are no longer in the list. * * `originInfo` is the OpenAPI `info` block (title/description/version) when @@ -84,9 +88,12 @@ export interface RegisterOriginResult { * APIs whose homepage isn't HTML, so the scraper has nothing to extract. */ export async function registerResourcesFromDiscovery( - resources: { url: string; authMode?: AuthMode }[], + resources: { url: string; method?: string; authMode?: AuthMode }[], source: string | undefined, - originInfo?: { title: string; description?: string } + originInfo?: { title: string; description?: string }, + /** Server-side probe session ID. URLs with a cached probe result in Redis + * skip re-probing — avoids rate limiting on registration. */ + probeSessionId?: string ): Promise { const originResourceCounts = new Map( await Promise.all( @@ -98,47 +105,79 @@ export async function registerResourcesFromDiscovery( ) ); + async function registerAsSiwx(resourceUrl: string) { + const siwxResult = await registerSiwxResource(resourceUrl, { + originMetadataFallback: originInfo, + }); + return siwxResult.success + ? { + success: true as const, + siwx: true as const, + url: resourceUrl, + resource: siwxResult.resource, + } + : { + success: false as const, + url: resourceUrl, + error: siwxResult.error, + }; + } + const results = await mapSettledWithConcurrency(resources, async resource => { const resourceUrl = resource.url; if (resource.authMode === 'siwx') { - return { - success: true as const, - siwx: true as const, - url: resourceUrl, - }; + return registerAsSiwx(resourceUrl); } - const probeResult = await probeX402Endpoint(resourceUrl); + // Check server-side probe cache (from the batch test). This skips + // re-probing and avoids rate limiting. The cache is server-authoritative + // — advisory data never round-trips through the client. + const cached = probeSessionId + ? await getCachedProbeResult(probeSessionId, resourceUrl) + : null; + let advisory: EndpointMethodAdvisory; + let probeWarnings: AuditWarning[] = []; - if (!probeResult.success) { - return { - success: false as const, - url: resourceUrl, - error: probeResult.error, - ...(probeResult.skipped ? { skipped: true as const } : {}), - ...(probeResult.statusCode !== undefined - ? { status: probeResult.statusCode } - : {}), - }; - } + if (cached) { + advisory = cached.advisory; + probeWarnings = cached.warnings; + } else { + const probeResult = await probeX402Endpoint(resourceUrl, resource.method); + + if (!probeResult.success) { + return { + success: false as const, + url: resourceUrl, + error: probeResult.error, + ...(probeResult.skipped ? { skipped: true as const } : {}), + ...(probeResult.statusCode !== undefined + ? { status: probeResult.statusCode } + : {}), + }; + } - const { advisory } = probeResult; + advisory = probeResult.advisory; + + // Drop discovery-level schema warnings superseded by other checks. + probeWarnings = probeResult.warnings.filter(w => { + if (w.code === 'SCHEMA_INPUT_MISSING' && advisory.inputSchema) + return false; + if (w.code === 'SCHEMA_OUTPUT_MISSING') return false; + return true; + }); + } // v1 rejection is handled inside registerResource() — no duplicate check needed here. if (advisory.authMode === 'siwx') { - return { - success: true as const, - siwx: true as const, - url: resourceUrl, - }; + return registerAsSiwx(resourceUrl); } const result = await registerResource(resourceUrl, advisory, { notifyNewServer: false, originMetadataFallback: originInfo, - warnings: probeResult.warnings, + warnings: probeWarnings, }); if (result.success) return result; @@ -176,6 +215,10 @@ export async function registerResourcesFromDiscovery( if ('success' in value && value.success) { if ('siwx' in value && value.siwx === true) { siwxResults.push({ url: resourceUrl }); + // Extract originId from SIWX registration result + if (!originId && 'resource' in value && value.resource?.origin?.id) { + originId = value.resource.origin.id; + } } else if ('resource' in value) { successfulResults.push({ url: resourceUrl, diff --git a/apps/scan/src/lib/discovery/utils/deduplicate-warnings.ts b/apps/scan/src/lib/discovery/utils/deduplicate-warnings.ts new file mode 100644 index 000000000..f557bb6e1 --- /dev/null +++ b/apps/scan/src/lib/discovery/utils/deduplicate-warnings.ts @@ -0,0 +1,12 @@ +import type { AuditWarning } from '@agentcash/discovery'; + +/** Deduplicate warnings by code + message, preserving order. */ +export function deduplicateWarnings(warnings: AuditWarning[]): AuditWarning[] { + const seen = new Set(); + return warnings.filter(w => { + const key = `${w.code}:${w.message}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} diff --git a/apps/scan/src/lib/discovery/utils/index.ts b/apps/scan/src/lib/discovery/utils/index.ts index 740ca3f3d..673fe285c 100644 --- a/apps/scan/src/lib/discovery/utils/index.ts +++ b/apps/scan/src/lib/discovery/utils/index.ts @@ -1,3 +1,4 @@ +export { deduplicateWarnings } from './deduplicate-warnings'; export { isX402PaymentOption } from './is-x402-option'; export { getRegistrationErrorMessage } from './registration-error-message'; export { PROBE_TIMEOUT_MS } from './constants'; diff --git a/apps/scan/src/lib/resources.ts b/apps/scan/src/lib/resources.ts index 65a1afdc4..7d772b08b 100644 --- a/apps/scan/src/lib/resources.ts +++ b/apps/scan/src/lib/resources.ts @@ -26,9 +26,11 @@ import { type ParsedX402Response, } from '@/lib/x402'; +import { scanDb } from '@x402scan/scan-db'; import type { AcceptsNetwork } from '@x402scan/scan-db'; import { convertOpenApiSchemaToV1 } from '@/lib/openapi-to-v1'; +import { deduplicateWarnings } from '@/lib/discovery/utils'; import { notifyNewServer } from '@/lib/discord-notifications'; /** @@ -73,13 +75,33 @@ export function validateResource( return { valid: false, error: 'No x402 payment options found' }; } - // Missing input schema + // Missing input schema — check advisory.inputSchema first, then fall back to + // the bazaar extension in the raw 402 body. The discovery package doesn't + // always extract schemas from bazaar, but the data is often there. if (!advisory.inputSchema) { - return { - valid: false, - error: - 'Missing input schema — add request/response schemas to your OpenAPI spec', - }; + let hasBazaarSchema = false; + if ( + advisory.paymentRequiredBody && + typeof advisory.paymentRequiredBody === 'object' && + advisory.paymentRequiredBody !== null + ) { + try { + const parsed = parseX402Response(advisory.paymentRequiredBody); + if (parsed.success) { + const extracted = getOutputSchema(parsed.data); + hasBazaarSchema = !!extracted; + } + } catch { + // Malformed bazaar data — treat as missing schema + } + } + if (!hasBazaarSchema) { + return { + valid: false, + error: + 'Missing input schema — add a requestBody or parameter schema to your OpenAPI spec so agents know what to send', + }; + } } // Unsupported networks @@ -98,19 +120,168 @@ export function validateResource( }; } - // SIWX warning — identity-gated endpoints are valid but not payment-protected - if (advisory.authMode === 'siwx') { - warnings.push({ - code: 'SIWX_ENDPOINT', - severity: 'warn', - message: - 'This endpoint uses SIWX authentication (identity-gated, not payment-protected)', - }); + // Missing output schema — endpoint works but agents won't know what it returns + if (!advisory.outputSchema) { + let hasBazaarOutputSchema = false; + if ( + advisory.paymentRequiredBody && + typeof advisory.paymentRequiredBody === 'object' && + advisory.paymentRequiredBody !== null + ) { + try { + const parsed = parseX402Response(advisory.paymentRequiredBody); + if (parsed.success) { + const extracted = getOutputSchema(parsed.data); + hasBazaarOutputSchema = !!extracted; + } + } catch { + // Malformed bazaar data + } + } + if (!hasBazaarOutputSchema) { + warnings.push({ + code: 'MISSING_OUTPUT_SCHEMA', + severity: 'warn', + message: + 'Missing output schema — add a response schema to your OpenAPI spec so agents know what this endpoint returns', + }); + } } return { valid: true, warnings }; } +/** + * Register a SIWX (free, identity-gated) endpoint. These endpoints have no + * x402 payment options, no 402 response body, and no Accepts records — just + * a Resource row linked to its Origin. + */ +export async function registerSiwxResource( + url: string, + options: { + originMetadataFallback?: { title?: string; description?: string }; + } = {} +) { + const urlObj = new URL(url); + if ( + urlObj.protocol === 'http:' && + urlObj.hostname !== 'localhost' && + urlObj.hostname !== '127.0.0.1' + ) { + return { + success: false as const, + error: 'HTTPS is required for resource registration', + }; + } + + urlObj.search = ''; + const cleanUrl = urlObj.toString(); + const origin = getOriginFromUrl(cleanUrl); + + try { + const resource = await scanDb.$transaction(async tx => { + await tx.resourceOrigin.upsert({ + where: { origin }, + create: { origin }, + update: {}, + }); + + return tx.resources.upsert({ + where: { resource: cleanUrl }, + create: { + resource: cleanUrl, + type: 'http', + x402Version: 0, + lastUpdated: new Date(), + metadata: { authMode: 'siwx' }, + origin: { connect: { origin } }, + }, + update: { + type: 'http', + x402Version: 0, + lastUpdated: new Date(), + metadata: { authMode: 'siwx' }, + deprecatedAt: null, + origin: { connect: { origin } }, + }, + include: { origin: true }, + }); + }); + + // Scrape and upsert origin metadata (non-blocking — resource is already + // persisted, so a scrape failure shouldn't fail the registration). + void (async () => { + try { + const { og, metadata, favicon } = await scrapeOriginData(origin); + const title = + metadata?.title ?? + og?.ogTitle ?? + options.originMetadataFallback?.title ?? + null; + const description = + metadata?.description ?? + og?.ogDescription ?? + options.originMetadataFallback?.description ?? + null; + + await upsertOrigin({ + origin, + title: title ?? undefined, + description: description ?? undefined, + favicon: favicon ?? undefined, + ogImages: + og?.ogImage?.map(image => ({ + url: image.url, + height: image.height, + width: image.width, + title: og.ogTitle, + description: og.ogDescription, + })) ?? [], + }); + } catch (err) { + console.error( + '[registerSiwxResource] Metadata scrape failed (non-blocking):', + err + ); + } + })(); + + return { + success: true as const, + resource: { + id: resource.id, + origin: { id: resource.origin.id }, + }, + }; + } catch (error) { + // P2002: unique constraint race — another concurrent call already registered + // the same URL (e.g. POST and DELETE on the same path). Treat as success. + const isUniqueViolation = + error instanceof Error && + 'code' in error && + (error as { code: string }).code === 'P2002'; + if (isUniqueViolation) { + const existing = await scanDb.resources.findUnique({ + where: { resource: cleanUrl }, + include: { origin: true }, + }); + // Record may have been deleted between the P2002 and the lookup — + // still treat as success since the constraint proved it existed. + return { + success: true as const, + resource: existing + ? { id: existing.id, origin: { id: existing.origin.id } } + : { id: 'race-resolved', origin: { id: 'race-resolved' } }, + }; + } + console.error('[registerSiwxResource] Failed:', error); + return { + success: false as const, + error: error instanceof Error ? error.message : 'Database error', + }; + } +} + export const registerResource = async ( url: string, advisory: EndpointMethodAdvisory, @@ -143,10 +314,10 @@ export const registerResource = async ( const x402Options = (advisory.paymentOptions ?? []).filter( isX402PaymentOption ); - const warnings: AuditWarning[] = [ + const warnings = deduplicateWarnings([ ...(options.warnings ?? []), ...validation.warnings, - ]; + ]); const urlObj = new URL(url); urlObj.search = ''; @@ -293,7 +464,12 @@ export const registerResource = async ( options.originMetadataFallback?.description ?? null; - await upsertOrigin({ + // Origin metadata upsert — non-blocking. The origin row itself is already + // created inside upsertResource's transaction. This just enriches it with + // scraped metadata (title, favicon, OG images). When multiple resources + // from the same origin register concurrently, this can race (P2002) — + // safe to swallow since another concurrent call will succeed. + void upsertOrigin({ origin, title: title ?? undefined, description: description ?? undefined, @@ -306,6 +482,8 @@ export const registerResource = async ( title: og.ogTitle, description: og.ogDescription, })) ?? [], + }).catch(() => { + // P2002 or other race — another call already upserted this origin. }); await upsertResourceResponse( diff --git a/apps/scan/src/services/db/resources/origin.ts b/apps/scan/src/services/db/resources/origin.ts index f30fd396a..c142f384f 100644 --- a/apps/scan/src/services/db/resources/origin.ts +++ b/apps/scan/src/services/db/resources/origin.ts @@ -29,12 +29,17 @@ function getDisplayableAcceptsWhere({ const displayableResourceWhere: Prisma.ResourcesWhereInput = { deprecatedAt: null, - response: { - isNot: null, - }, - accepts: { - some: getDisplayableAcceptsWhere({}), - }, + OR: [ + // Paid resources: must have a stored 402 response and supported accepts + { + response: { isNot: null }, + accepts: { some: getDisplayableAcceptsWhere({}) }, + }, + // SIWX (free) resources: identified by metadata.authMode + { + metadata: { path: ['authMode'], equals: 'siwx' }, + }, + ], }; const ogImageSchema = z.object({ @@ -132,16 +137,21 @@ export const listOriginsSchema = z.object({ export const listOrigins = async (input: z.infer) => { const { chain, address } = input; const acceptsWhere = getDisplayableAcceptsWhere({ chain, address }); + // SIWX (free) resources have no chain/address — only include them when + // no chain or address filter is applied. + const hasPaymentFilter = chain != null || address != null; + const resourceFilter: Prisma.ResourcesWhereInput = hasPaymentFilter + ? { deprecatedAt: null, accepts: { some: acceptsWhere } } + : { + deprecatedAt: null, + OR: [ + { accepts: { some: acceptsWhere } }, + { metadata: { path: ['authMode'], equals: 'siwx' } }, + ], + }; const origins = await scanDb.resourceOrigin.findMany({ where: { - resources: { - some: { - deprecatedAt: null, - accepts: { - some: acceptsWhere, - }, - }, - }, + resources: { some: resourceFilter }, }, orderBy: { createdAt: 'desc' }, }); @@ -159,32 +169,30 @@ export const listOriginsWithResources = async ( ) => { const { chain, address, originIds } = input; const acceptsWhere = getDisplayableAcceptsWhere({ chain, address }); + // SIWX (free) resources have no chain/address — only include them when + // no chain or address filter is applied. + const hasPaymentFilter = chain != null || address != null; + const paidOrSiwxResource: Prisma.ResourcesWhereInput = hasPaymentFilter + ? { + deprecatedAt: null, + response: { isNot: null }, + accepts: { some: acceptsWhere }, + } + : { + deprecatedAt: null, + OR: [ + { response: { isNot: null }, accepts: { some: acceptsWhere } }, + { metadata: { path: ['authMode'], equals: 'siwx' } }, + ], + }; const origins = await scanDb.resourceOrigin.findMany({ where: { ...(originIds ? { id: { in: originIds } } : {}), - resources: { - some: { - deprecatedAt: null, - response: { - isNot: null, - }, - accepts: { - some: acceptsWhere, - }, - }, - }, + resources: { some: paidOrSiwxResource }, }, include: { resources: { - where: { - deprecatedAt: null, - response: { - isNot: null, - }, - accepts: { - some: acceptsWhere, - }, - }, + where: paidOrSiwxResource, orderBy: { resource: 'asc', }, diff --git a/apps/scan/src/services/db/resources/resource.ts b/apps/scan/src/services/db/resources/resource.ts index 2f312003d..70f3f2183 100644 --- a/apps/scan/src/services/db/resources/resource.ts +++ b/apps/scan/src/services/db/resources/resource.ts @@ -40,7 +40,16 @@ export const upsertResource = async ( return; } const originStr = getOriginFromUrl(baseResource.resource); + return await scanDb.$transaction(async tx => { + // Ensure the origin exists inside the transaction to avoid + // concurrent connectOrCreate race conditions (P2002). + await tx.resourceOrigin.upsert({ + where: { origin: originStr }, + create: { origin: originStr }, + update: {}, + }); + const { origin, ...resource } = await tx.resources.upsert({ where: { resource: baseResource.resource, @@ -52,13 +61,8 @@ export const upsertResource = async ( lastUpdated: baseResource.lastUpdated, metadata: baseResource.metadata, origin: { - connectOrCreate: { - where: { - origin: originStr, - }, - create: { - origin: originStr, - }, + connect: { + origin: originStr, }, }, }, @@ -288,18 +292,21 @@ const searchResourcesUncached = async ( showDeprecated, chains, } = input; + const acceptsFilter: Prisma.AcceptsWhereInput = + chains !== undefined ? { network: { in: chains } } : {}; return await scanDb.resources.findMany({ where: { - accepts: { - some: - chains !== undefined - ? { - network: { - in: chains, - }, - } - : {}, - }, + // Include paid resources (with accepts) and SIWX (free) resources. + // When filtering by chain, SIWX resources are excluded since they + // have no chain-specific payment options. + ...(chains !== undefined + ? { accepts: { some: acceptsFilter } } + : { + OR: [ + { accepts: { some: {} } }, + { metadata: { path: ['authMode'], equals: 'siwx' } }, + ], + }), ...(search ? { OR: [ diff --git a/apps/scan/src/services/discovery/fetch-discovery.ts b/apps/scan/src/services/discovery/fetch-discovery.ts index 9819b30bd..478102af5 100644 --- a/apps/scan/src/services/discovery/fetch-discovery.ts +++ b/apps/scan/src/services/discovery/fetch-discovery.ts @@ -62,10 +62,16 @@ export async function fetchDiscoveryDocument( }; } + const expectedOrigin = new URL(discovered.origin).origin; const resources: DiscoveredResource[] = discovered.endpoints.flatMap( endpoint => { try { - const url = new URL(endpoint.path, discovered.origin).toString(); + const resolved = new URL(endpoint.path, discovered.origin); + // Security: reject endpoints that resolve to a different origin. + // A malicious OpenAPI spec could contain absolute URLs pointing + // elsewhere (new URL('https://evil.com/x', base) ignores the base). + if (resolved.origin !== expectedOrigin) return []; + const url = resolved.toString(); return [ { url, diff --git a/apps/scan/src/trpc/routers/developer.ts b/apps/scan/src/trpc/routers/developer.ts index 79ba86049..e1d15c2d0 100644 --- a/apps/scan/src/trpc/routers/developer.ts +++ b/apps/scan/src/trpc/routers/developer.ts @@ -8,6 +8,11 @@ import type { FailedResource, TestedResource } from '@/types/batch-test'; import { probeX402Endpoint } from '@/lib/discovery/probe'; import { validateResource } from '@/lib/resources'; import { fetchDiscoveryDocument } from '@/services/discovery'; +import { deduplicateWarnings } from '@/lib/discovery/utils'; +import { + createProbeSession, + cacheProbeResult, +} from '@/lib/discovery/probe-cache'; /** * Test a single resource by probing it and running the same validation @@ -70,13 +75,26 @@ async function testSingleResource( }; } + // Drop discovery-level schema warnings superseded by other checks. + // SCHEMA_INPUT_MISSING: suppressed when advisory already has inputSchema + // from OpenAPI (the 402 body lacking it is not actionable). + // SCHEMA_OUTPUT_MISSING: always suppressed — validateResource() has its + // own MISSING_OUTPUT_SCHEMA check that includes bazaar fallback, so + // the discovery-level warning would be a duplicate or less precise. + const probeWarnings = result.warnings.filter(w => { + if (w.code === 'SCHEMA_INPUT_MISSING' && advisory.inputSchema) + return false; + if (w.code === 'SCHEMA_OUTPUT_MISSING') return false; + return true; + }); + return { success: true as const, url, method: advisory.method as TestedResource['method'], description: advisory.summary ?? null, parsed: advisory, - warnings: [...result.warnings, ...validation.warnings], + warnings: deduplicateWarnings([...probeWarnings, ...validation.warnings]), }; } catch (err) { return { @@ -132,10 +150,15 @@ export const developerRouter = createTRPCRouter({ }; }), - /** Batch test multiple resources to get their x402 responses */ + /** Batch test multiple resources to get their x402 responses. + * Successful probe results are cached server-side under `probeSessionId` + * so the registration endpoint can reuse them without trusting client data. */ batchTest: publicProcedure .input( z.object({ + /** Probe session ID. If omitted, a new session is created. Pass the + * sessionId from a previous batch to append to the same cache. */ + probeSessionId: z.string().uuid().optional(), resources: z .array( z.object({ @@ -171,6 +194,8 @@ export const developerRouter = createTRPCRouter({ }) ) .mutation(async ({ input }) => { + const sessionId = input.probeSessionId ?? createProbeSession(); + // Separate invalid resources from valid ones const invalidResults: FailedResource[] = input.resources .filter(r => r.invalid) @@ -180,22 +205,37 @@ export const developerRouter = createTRPCRouter({ error: r.invalidReason ?? 'Invalid resource format', })); - // SIWX routes are identity-gated, not payment-protected. Skip probing - // them — they are surfaced via authModeMap on the client and should not - // appear in either the tested or failed buckets. + // Probe endpoints that are x402-paid or unclassified (might be paid but + // discovery didn't detect it). Skip SIWX (identity-gated, not x402) and + // explicitly non-paid (unprotected, apiKey). + const skipModes = new Set(['siwx', 'unprotected', 'apiKey']); const probeableResources = input.resources.filter( - r => !r.invalid && r.authMode !== 'siwx' - ); - const testResults = await Promise.all( - probeableResources.map(r => - testSingleResource(r.url, r.method, r.sampleBody) - ) + r => !r.invalid && (r.authMode == null || !skipModes.has(r.authMode)) ); + // Probe sequentially to avoid overwhelming the merchant's server. + // Concurrent probes to the same origin can trigger rate limiting (503s). + const testResults = []; + for (const r of probeableResources) { + const result = await testSingleResource(r.url, r.method, r.sampleBody); + // Cache successful probes server-side so registration can reuse them + // without the advisory data ever round-tripping through the client. + if (result.success) { + void cacheProbeResult( + sessionId, + result.url, + result.parsed, + result.warnings ?? [] + ); + } + testResults.push(result); + } - // Combine test results with invalid results + // Combine test results with invalid results only. Non-paid endpoints + // are handled client-side (grey strikethrough) — they're not errors. const allResults = [...testResults, ...invalidResults]; return { + probeSessionId: sessionId, resources: allResults.filter( (r): r is Extract => r.success ), diff --git a/apps/scan/src/trpc/routers/public/resources.ts b/apps/scan/src/trpc/routers/public/resources.ts index c75077931..e5457ae46 100644 --- a/apps/scan/src/trpc/routers/public/resources.ts +++ b/apps/scan/src/trpc/routers/public/resources.ts @@ -144,11 +144,19 @@ export const resourcesRouter = createTRPCRouter({ /** * Register all x402 resources discovered from an origin. * Uses DNS TXT records (_x402.{hostname}) or /.well-known/x402 for discovery. + * + * SECURITY: Advisory data (payTo addresses, schemas, etc.) is never accepted + * from the client. If a probeSessionId is provided, the server reads cached + * probe results that it wrote during the batch test. This prevents payment + * redirection attacks where a malicious client substitutes payTo addresses. */ registerFromOrigin: publicProcedure .input( z.object({ origin: z.url(), + /** Server-side probe session from the batch test. URLs with a cached + * probe result skip re-probing — avoids rate limiting on registration. */ + probeSessionId: z.string().uuid().optional(), }) ) .mutation(async ({ input }) => { @@ -167,16 +175,17 @@ export const resourcesRouter = createTRPCRouter({ const result = await registerResourcesFromDiscovery( discoveryResult.resources, discoveryResult.source, - discoveryResult.info + discoveryResult.info, + input.probeSessionId ); - if (result.registered === 0) { + if (result.registered === 0 && result.siwx === 0) { return { success: false as const, error: { type: 'noValidResources' as const, message: - 'No valid paid x402 resources were found for this origin. Add at least one paid x402 resource that passes validation to complete registration.', + 'No valid x402 or free (SIWX) resources were found for this origin. Add at least one resource that passes validation to complete registration.', }, result, };