diff --git a/defi/package.json b/defi/package.json index 0f87bada1d..2809d82258 100644 --- a/defi/package.json +++ b/defi/package.json @@ -59,7 +59,8 @@ "twitter-update": "npx ts-node --logError --transpile-only src/twitter/updateTwitterData.ts", "local-test-por": "npx ts-node --logError --transpile-only proof-of-reserves/cli/test.ts", "run-check-por": "npx ts-node --logError --transpile-only proof-of-reserves/cli/check.ts", - "ui-tool": "npm run update-submodules; cd ui-tool; npm run start-server" + "ui-tool": "npm run update-submodules; cd ui-tool; npm run start-server", + "validate-api": "npx ts-node --logError --transpile-only src/utils/validateAPI.ts" }, "devDependencies": { "@babel/preset-env": "^7.18.2", @@ -93,6 +94,7 @@ "@defillama/sdk": "^5.0.172", "@elastic/elasticsearch": "^8.13.1", "@supercharge/promise-pool": "^2.3.2", + "@types/ajv": "^0.0.5", "@types/async-retry": "^1.4.8", "@types/aws-lambda": "^8.10.97", "@types/inquirer-autocomplete-prompt": "^2.0.0", @@ -101,6 +103,7 @@ "@types/node": "^18.0.0", "@types/node-fetch": "^2.5.10", "@types/promise.allsettled": "^1.0.6", + "ajv": "^8.17.1", "axios": "^1.6.5", "bignumber.js": "^9.0.1", "buffer-layout": "^1.2.2", diff --git a/defi/src/utils/apiValidateUtils.ts b/defi/src/utils/apiValidateUtils.ts new file mode 100644 index 0000000000..75227ca124 --- /dev/null +++ b/defi/src/utils/apiValidateUtils.ts @@ -0,0 +1,1033 @@ +import fs from 'fs'; +import path from 'path'; +import axios, { AxiosResponse } from 'axios'; +import Ajv from 'ajv'; + +export interface SchemaInfo { + schema: any; + serverUrl: string; +} + +export interface EndpointInfo { + path: string; + serverUrl: string; + schema: any; + method: string; + queryParams?: Record; + override?: EndpointOverride; +} + +export interface ValidationResult { + endpoint: string; + serverUrl: string; + status: 'pass' | 'fail' | 'expected_failure'; + errors: string[]; + responseTime?: number; + override?: EndpointOverride; + queryParams?: Record; +} + +const schemasCache = new Map(); +const ajv = new Ajv({ + allErrors: true, + verbose: true, + strict: false, + removeAdditional: false, + coerceTypes: true +}); + +//handle uint +ajv.addFormat('uint', { + type: 'number', + validate: (value: number) => { + return Number.isInteger(value) && value >= 0; + } +}); + + +export interface EndpointOverride { + parameterOverrides?: Record; + skip?: boolean; + expectedFailure?: boolean; + skipDataComparison?: boolean; + reason?: string; +} + +// Endpoint overrides for special cases +// Note: For compareBeta (prod vs beta), data comparison is enabled by default +// since both environments use the same DB and should return matching values. +// For validateAPI (single environment), only schema is validated. +export const ENDPOINT_OVERRIDES: Record = { + // Add specific overrides here if needed (e.g., skip certain endpoints entirely) +}; + +export async function fetchWithRetry( + url: string, + retries: number = 3, + delay: number = 1000, + timeout: number = 45000 +): Promise { + for (let i = 0; i <= retries; i++) { + try { + const response = await axios.get(url, { + timeout, + headers: { + 'User-Agent': 'DefiLlama-API-Validator/1.0' + } + }); + return response; + } catch (error: any) { + const isLastAttempt = i === retries; + const shouldRetry = shouldRetryError(error); + + if (isLastAttempt || !shouldRetry) { + throw error; + } + + const backoffDelay = calculateBackoffDelay(delay, i, error); + await new Promise(resolve => setTimeout(resolve, backoffDelay)); + } + } + throw new Error('Unexpected error in fetchWithRetry'); +} + +function shouldRetryError(error: any): boolean { + if (error.code) { + const retryableCodes = [ + 'ECONNRESET', + 'ETIMEDOUT', + 'ENOTFOUND', + 'ECONNREFUSED', + 'EHOSTUNREACH', + 'EPIPE', + 'EAI_AGAIN' + ]; + + if (retryableCodes.includes(error.code)) { + return true; + } + } + + if (error.response?.status) { + const retryableStatuses = [ + 429, + 502, + 503, + 504, + 520, + 521, + 522, + 523, + 524 + ]; + + return retryableStatuses.includes(error.response.status); + } + + if (error.message?.toLowerCase().includes('timeout')) { + return true; + } + + return false; +} + +function calculateBackoffDelay(baseDelay: number, attempt: number, error: any): number { + let backoffDelay = baseDelay * Math.pow(2, attempt); + if (error.response?.status === 429) { + const retryAfter = error.response.headers['retry-after']; + if (retryAfter) { + const retryAfterMs = parseInt(retryAfter) * 1000; + backoffDelay = Math.max(backoffDelay, retryAfterMs); + } else { + backoffDelay = baseDelay * Math.pow(3, attempt); + } + } + + const jitter = backoffDelay * 0.25 * (Math.random() - 0.5); + backoffDelay += jitter; + + // Ensure delay is between 1-5 seconds + return Math.max(1000, Math.min(backoffDelay, 5000)); +} + +export async function loadOpenApiSpec(apiType: 'free' | 'pro' = 'free'): Promise { + const cacheKey = `openapi-${apiType}`; + + if (schemasCache.has(cacheKey)) { + return schemasCache.get(cacheKey); + } + + const githubUrl = apiType === 'pro' + ? 'https://raw.githubusercontent.com/DefiLlama/api-docs/refs/heads/main/defillama-openapi-pro.json' + : 'https://raw.githubusercontent.com/DefiLlama/api-docs/refs/heads/main/defillama-openapi-free.json'; + + try { + console.log(`getting spec from api-docs for ${apiType}`); + const response = await axios.get(githubUrl, { + timeout: 30000, + }); + + const spec = response.data; + schemasCache.set(cacheKey, spec); + return spec; + + } catch (error: any) { + throw new Error(`spec fetch failed: ${error.message}`); + } +} + +export function getServerUrl(endpoint: string, spec: any, apiType: 'free' | 'pro' = 'free'): string { + try { + const endpointDef = spec.paths[endpoint]; + let serverUrl: string; + + if (!endpointDef) { + serverUrl = spec.servers?.[0]?.url || process.env.BASE_API_URL || 'https://api.llama.fi'; + } else { + const getMethod = endpointDef.get; + if (getMethod?.servers?.length > 0) { + serverUrl = getMethod.servers[0].url; + } else { + serverUrl = spec.servers?.[0]?.url || process.env.BASE_API_URL || 'https://api.llama.fi'; + } + } + + if (apiType === 'pro' && process.env.PRO_API_KEY && serverUrl.includes('pro-api.llama.fi')) { + serverUrl = `${serverUrl}/${process.env.PRO_API_KEY}`; + } + + return serverUrl; + } catch (error) { + const baseUrl = process.env.BASE_API_URL || 'https://api.llama.fi'; + + if (apiType === 'pro' && process.env.PRO_API_KEY && baseUrl.includes('pro-api.llama.fi')) { + return `${baseUrl}/${process.env.PRO_API_KEY}`; + } + + return baseUrl; + } +} + +export function getBetaServerUrl(prodServerUrl: string): string { + try { + const prodUrl = new URL(prodServerUrl); + const path = prodUrl.pathname; + + let betaBaseUrl = ''; + + if (prodUrl.hostname === 'coins.llama.fi') { + betaBaseUrl = process.env.BETA_COINS_URL || process.env.BETA_API_URL || ''; + } else if (prodUrl.hostname === 'stablecoins.llama.fi') { + betaBaseUrl = process.env.BETA_STABLECOINS_URL || process.env.BETA_API_URL || ''; + } else if (prodUrl.hostname === 'yields.llama.fi') { + betaBaseUrl = process.env.BETA_YIELDS_URL || process.env.BETA_API_URL || ''; + } else if (prodUrl.hostname === 'api.llama.fi') { + betaBaseUrl = process.env.BETA_API_URL || ''; + } else if (prodUrl.hostname === 'pro-api.llama.fi') { + betaBaseUrl = process.env.BETA_PRO_API_URL || process.env.BETA_API_URL || ''; + } else if (prodUrl.hostname === 'bridges.llama.fi') { + betaBaseUrl = process.env.BETA_BRIDGES_URL || process.env.BETA_API_URL || ''; + } else { + console.warn(`No beta URL mapping found for domain: ${prodUrl.hostname}`); + betaBaseUrl = process.env.BETA_API_URL || ''; + } + + if (!betaBaseUrl) { + console.warn(`Beta URL not configured for ${prodUrl.hostname}`); + return ''; + } + + const cleanBaseUrl = betaBaseUrl.replace(/\/$/, ''); + const cleanPath = path.startsWith('/') ? path : '/' + path; + + return cleanBaseUrl + cleanPath + prodUrl.search; + + } catch (error) { + console.error(`Error parsing URL ${prodServerUrl}:`, error); + return process.env.BETA_API_URL || ''; + } +} + +function makeFieldsNullable(schema: any, isRoot: boolean = false): any { + if (!schema || typeof schema !== 'object') { + return schema; + } + + if (isRoot) { + if (schema.type === 'array') { + return { + ...schema, + items: schema.items ? makeFieldsNullable(schema.items) : schema.items + }; + } + if (schema.type === 'object') { + const newProperties: any = {}; + if (schema.properties) { + for (const [key, value] of Object.entries(schema.properties)) { + newProperties[key] = makeFieldsNullable(value); + } + } + return { + ...schema, + properties: newProperties, + additionalProperties: schema.additionalProperties ? makeFieldsNullable(schema.additionalProperties) : schema.additionalProperties + }; + } + } + + if (schema.type === 'number') { + return { + ...schema, + type: ['number', 'string', 'null'] //sometime we return numbers as strings + }; + } + + if (Array.isArray(schema.type) && schema.type.includes('number') && schema.type.includes('null')) { + return { + ...schema, + type: ['number', 'string', 'null'] + }; + } + + //make strings flexible to accept arrays and numbers (for fields like governanceID, valuation) + if (schema.type === 'string') { + return { + ...schema, + type: ['string', 'array', 'number', 'null'] + }; + } + + //make arrays nullable and process their items + if (schema.type === 'array') { + return { + ...schema, + type: ['array', 'null'], + items: schema.items ? makeFieldsNullable(schema.items) : schema.items + }; + } + + //handle objects with properties - make them nullable + if (schema.type === 'object' && schema.properties) { + const newProperties: any = {}; + for (const [key, value] of Object.entries(schema.properties)) { + newProperties[key] = makeFieldsNullable(value); + } + return { + ...schema, + type: ['object', 'null'], + properties: newProperties, + additionalProperties: true + }; + } + + //handle objects with additionalProperties (like chainTvls) - make them nullable too + if (schema.type === 'object' && schema.additionalProperties) { + return { + ...schema, + type: ['object', 'null'], + additionalProperties: makeFieldsNullable(schema.additionalProperties) + }; + } + + //handle plain objects without properties or additionalProperties - make them very flexible + if (schema.type === 'object' && !schema.properties && !schema.additionalProperties) { + return { + ...schema, + type: ['object', 'null', 'string'] + }; + } + + //handle anyOf, oneOf, allOf - make them more permissive + if (schema.anyOf) { + return { + ...schema, + anyOf: schema.anyOf.map((s: any) => makeFieldsNullable(s)) + }; + } + + if (schema.oneOf) { + const oneOfTypes = schema.oneOf.map((s: any) => makeFieldsNullable(s)); + oneOfTypes.push({ type: ['string', 'number', 'object', 'array', 'boolean', 'null'] }); + return { + ...schema, + anyOf: oneOfTypes //use anyOf instead of oneOf for more permissive validation + }; + } + + if (schema.allOf) { + return { + ...schema, + allOf: schema.allOf.map((s: any) => makeFieldsNullable(s)) + }; + } + + return schema; +} + +export function extractSchemaFromOpenApi(endpoint: string, spec: any): any | null { + try { + const endpointDef = spec.paths[endpoint]; + if (!endpointDef?.get?.responses?.['200']?.content?.['application/json']?.schema) { + return null; + } + + const rawSchema = endpointDef.get.responses['200'].content['application/json'].schema; + return makeFieldsNullable(rawSchema, true); + } catch (error) { + console.error(`Error extracting schema for endpoint ${endpoint}:`, error); + return null; + } +} + +export interface QueryParameterInfo { + queryParams: Record; + hasRequiredParams: boolean; +} + +export function extractQueryParameters(endpointDef: any): QueryParameterInfo { + const queryParams: Record = {}; + let hasRequiredParams = false; + + if (!endpointDef.parameters) { + return { queryParams, hasRequiredParams }; + } + + endpointDef.parameters.forEach((param: any) => { + if (param.in === 'query') { + const paramName = param.name; + const schema = param.schema; + + if (param.required) { + hasRequiredParams = true; + } + + //use schema example if available + if (schema?.example !== undefined && schema?.example !== null) { + queryParams[paramName] = [String(schema.example)]; + } + //use schema default if available + else if (schema?.default !== undefined && schema?.default !== null) { + queryParams[paramName] = [String(schema.default)]; + } + //use schema enum if available + else if (schema?.enum && Array.isArray(schema.enum)) { + queryParams[paramName] = schema.enum.map(String); + } + } + }); + + return { queryParams, hasRequiredParams }; +} + +export function buildUrlWithQueryParams(baseUrl: string, queryParams: Record): string { + if (Object.keys(queryParams).length === 0) { + return baseUrl; + } + + const url = new URL(baseUrl); + Object.entries(queryParams).forEach(([key, value]) => { + url.searchParams.append(key, value); + }); + + return url.toString(); +} + +export function generateQueryParameterVariations(queryParams: Record, hasRequiredParams: boolean = false): Record[] { + if (Object.keys(queryParams).length === 0) { + return [{}]; + } + + const paramNames = Object.keys(queryParams); + const variations: Record[] = []; + const maxVariationsPerEndpoint = 10; + + function generateCombinations(currentParams: Record, remainingParamNames: string[]): void { + if (variations.length >= maxVariationsPerEndpoint) { + return; + } + + if (remainingParamNames.length === 0) { + variations.push({ ...currentParams }); + return; + } + + const paramName = remainingParamNames[0]; + const samples = queryParams[paramName]; + + samples.forEach(sample => { + if (variations.length < maxVariationsPerEndpoint) { + const newParams = { ...currentParams, [paramName]: sample }; + generateCombinations(newParams, remainingParamNames.slice(1)); + } + }); + } + + if (!hasRequiredParams) { + variations.push({}); + } + + generateCombinations({}, paramNames); + + return variations.slice(0, maxVariationsPerEndpoint); +} + +export function getEndpointOverride(endpoint: string): EndpointOverride | undefined { + if (ENDPOINT_OVERRIDES[endpoint]) { + return ENDPOINT_OVERRIDES[endpoint]; + } + + for (const [pattern, override] of Object.entries(ENDPOINT_OVERRIDES)) { + if (pattern.includes('{') && endpointMatchesPattern(endpoint, pattern)) { + return override; + } + } + + return undefined; +} + +function endpointMatchesPattern(endpoint: string, pattern: string): boolean { + const regexPattern = pattern.replace(/\{[^}]+\}/g, '[^/]+'); + const regex = new RegExp(`^${regexPattern}$`); + return regex.test(endpoint); +} + +export function substituteParameters(endpoint: string, spec: any, parameterValues?: Record): string[] { + const paramMatches = endpoint.match(/\{([^}]+)\}/g); + + if (!paramMatches) { + return [endpoint]; + } + + const override = getEndpointOverride(endpoint); + + const variations: string[] = []; + const paramNames = paramMatches.map(match => match.slice(1, -1)); + + const endpointDef = spec.paths[endpoint]; + const pathParameters = endpointDef?.get?.parameters?.filter((p: any) => p.in === 'path') || []; + + function generateCombinations(currentEndpoint: string, remainingParams: string[]): void { + if (remainingParams.length === 0) { + variations.push(currentEndpoint); + return; + } + + const paramName = remainingParams[0]; + + let samples: string[]; + + if (parameterValues?.[paramName]) { + samples = [parameterValues[paramName]]; + } else if (override?.parameterOverrides?.[paramName]) { + samples = override.parameterOverrides[paramName]; + } else { + const paramDef = pathParameters.find((p: any) => p.name === paramName); + + if (paramDef?.schema?.example !== undefined && paramDef?.schema?.example !== null) { + samples = [String(paramDef.schema.example)]; + } else if (paramDef?.schema?.default !== undefined && paramDef?.schema?.default !== null) { + samples = [String(paramDef.schema.default)]; + } else { + samples = ['sample']; + } + } + + samples.forEach(sample => { + const newEndpoint = currentEndpoint.replace(`{${paramName}}`, sample); + generateCombinations(newEndpoint, remainingParams.slice(1)); + }); + } + + generateCombinations(endpoint, paramNames); + return variations; +} + +export function getAllEndpoints(spec: any, includeParameterized: boolean = true, apiType: 'free' | 'pro' = 'free'): EndpointInfo[] { + const endpoints: EndpointInfo[] = []; + let skippedProEndpoints = 0; + + Object.entries(spec.paths).forEach(([path, methods]: [string, any]) => { + if (methods.get) { + // Filter endpoints based on API type + const endpointSecurity = methods.get.security; + const isPro = endpointSecurity && endpointSecurity.length > 0; + + // Skip pro endpoints when testing free API + if (apiType === 'free' && isPro) { + skippedProEndpoints++; + return; + } + + const serverUrl = getServerUrl(path, spec, apiType); + const schema = extractSchemaFromOpenApi(path, spec); + const { queryParams, hasRequiredParams } = extractQueryParameters(methods.get); + + if (schema) { + if (path.includes('{') && includeParameterized) { + const pathVariations = substituteParameters(path, spec); + pathVariations.forEach(variation => { + const override = getEndpointOverride(variation); + + if (override?.skip) { + return; + } + + const queryVariations = generateQueryParameterVariations(queryParams, hasRequiredParams); + + queryVariations.forEach(queryParamSet => { + endpoints.push({ + path: variation, + serverUrl, + schema, + method: 'GET', + queryParams: queryParamSet, + override + }); + }); + }); + } else if (!path.includes('{')) { + const override = getEndpointOverride(path); + + if (override?.skip) { + return; + } + + const queryVariations = generateQueryParameterVariations(queryParams, hasRequiredParams); + + queryVariations.forEach(queryParamSet => { + endpoints.push({ + path, + serverUrl, + schema, + method: 'GET', + queryParams: queryParamSet, + override + }); + }); + } + } + } + }); + + if (apiType === 'free' && skippedProEndpoints > 0) { + console.log(`Skipped ${skippedProEndpoints} pro-only endpoints (testing free API)`); + } + + return endpoints; +} + +export async function validateResponseAgainstSchema( + response: any, + schema: any, + isDebug: boolean = false +): Promise<{ valid: boolean; errors: string[] }> { + try { + if (isDebug) { + console.log('\n=== VALIDATION DEBUG ==='); + console.log('Response type:', typeof response); + console.log('Response keys:', response && typeof response === 'object' ? Object.keys(response) : 'N/A'); + console.log('Response preview:', JSON.stringify(response, null, 2).slice(0, 500) + '...'); + } + + let actualData = response; + + // Handle stringified entire response + if (typeof response === 'string') { + try { + actualData = JSON.parse(response); + if (isDebug) { + console.log('Parsed stringified response'); + } + } catch (parseError) { + if (isDebug) { + console.log('Response is string but not valid JSON:', parseError); + } + } + } + + // Handle stringified JSON in body field + if (actualData && typeof actualData === 'object' && actualData.body && typeof actualData.body === 'string') { + try { + const parsedBody = JSON.parse(actualData.body); + if (isDebug) { + console.log('Parsed body field'); + } + actualData = parsedBody; + } catch (parseError) { + if (isDebug) { + console.log('Failed to parse body field:', parseError); + } + } + } + + // Handle double-escaped JSON (fallback for when actualData is still a string) + if (typeof actualData === 'string') { + try { + // Try to parse again in case of double-escaping + actualData = JSON.parse(actualData); + if (isDebug) { + console.log('Parsed double-escaped JSON'); + } + } catch (parseError) { + // Keep as is + } + } + + if (isDebug) { + console.log('Schema expects type:', schema.type || 'Any'); + console.log('Schema properties:', schema.properties ? Object.keys(schema.properties) : 'N/A'); + console.log('========================\n'); + } + + const validate = ajv.compile(schema); + let valid = validate(actualData); + + // If validation failed and data is still a string, try one more unescape attempt + if (!valid && typeof actualData === 'string') { + try { + const unescaped = actualData.replace(/\\"/g, '"').replace(/\\\\/g, '\\'); + const parsedUnescaped = JSON.parse(unescaped); + actualData = parsedUnescaped; + valid = validate(actualData); + if (valid && isDebug) { + console.log('Successfully validated after unescaping'); + } + } catch (e) { + // Keep original validation result + } + } + + if (valid) { + return { valid: true, errors: [] }; + } else { + const errors = validate.errors?.map(error => { + const errorMsg = `${error.instancePath || '/body'}: ${error.message}`; + if (isDebug) { + console.log('Validation error details:', { + path: error.instancePath || '/body', + message: error.message, + data: error.data + }); + } + return errorMsg; + }) || ['Unknown validation error']; + + // DEBUG: Log full response on validation failure + console.log('\n[DEBUG] Schema validation failed. Full response data:'); + console.log(JSON.stringify(actualData, null, 2)); + console.log('\n[DEBUG] Expected schema:'); + console.log(JSON.stringify(schema, null, 2)); + console.log('[DEBUG] End of validation failure data\n'); + + return { valid: false, errors }; + } + } catch (error: any) { + if (isDebug) { + console.log('Schema validation exception:', error.message); + } + return { + valid: false, + errors: [`Schema validation error: ${error.message}`] + }; + } +} + +export async function testEndpoint( + endpointInfo: EndpointInfo, + isDebug: boolean = false, + retryCount: number = 3, + retryDelay: number = 1000, + requestTimeout: number = 45000 +): Promise { + const baseUrl = `${endpointInfo.serverUrl}${endpointInfo.path}`; + const fullUrl = buildUrlWithQueryParams(baseUrl, endpointInfo.queryParams || {}); + const startTime = Date.now(); + + try { + const response = await fetchWithRetry(fullUrl, retryCount, retryDelay, requestTimeout); + const responseTime = Date.now() - startTime; + + if (response.status !== 200) { + if (endpointInfo.override?.expectedFailure) { + return { + endpoint: endpointInfo.path, + serverUrl: endpointInfo.serverUrl, + status: 'expected_failure', + errors: [`Expected failure: ${endpointInfo.override.reason || 'No reason provided'}`], + responseTime, + override: endpointInfo.override, + queryParams: endpointInfo.queryParams + }; + } + + return { + endpoint: endpointInfo.path, + serverUrl: endpointInfo.serverUrl, + status: 'fail', + errors: [`HTTP ${response.status}: ${response.statusText}`], + responseTime, + override: endpointInfo.override, + queryParams: endpointInfo.queryParams + }; + } + + if (isDebug) { + console.log(`\n--- ENDPOINT DEBUG: ${endpointInfo.path} ---`); + console.log('Raw response status:', response.status); + console.log('Raw response headers content-type:', response.headers['content-type']); + console.log('Raw response data type:', typeof response.data); + console.log('Raw response data preview:', JSON.stringify(response.data, null, 2).slice(0, 400) + '...'); + console.log('Expected schema type:', endpointInfo.schema?.type); + console.log('--- END ENDPOINT DEBUG ---\n'); + } + + const validation = await validateResponseAgainstSchema(response.data, endpointInfo.schema, isDebug); + + return { + endpoint: endpointInfo.path, + serverUrl: endpointInfo.serverUrl, + status: validation.valid ? 'pass' : 'fail', + errors: validation.errors, + responseTime, + override: endpointInfo.override, + queryParams: endpointInfo.queryParams + }; + + } catch (error: any) { + const responseTime = Date.now() - startTime; + + if (endpointInfo.override?.expectedFailure) { + return { + endpoint: endpointInfo.path, + serverUrl: endpointInfo.serverUrl, + status: 'expected_failure', + errors: [`Expected failure: ${endpointInfo.override.reason || 'No reason provided'}`], + responseTime, + override: endpointInfo.override, + queryParams: endpointInfo.queryParams + }; + } + + let errorMessage = error.message || 'Unknown error'; + + if (error.code) { + errorMessage += ` (${error.code})`; + } + + if (error.response?.status) { + errorMessage += ` - HTTP ${error.response.status}`; + if (error.response.statusText) { + errorMessage += `: ${error.response.statusText}`; + } + } + + if (isDebug) { + console.log(`Network error for ${fullUrl}:`, errorMessage); + console.log('Error details:', { + code: error.code, + status: error.response?.status, + statusText: error.response?.statusText, + url: fullUrl + }); + } + + return { + endpoint: endpointInfo.path, + serverUrl: endpointInfo.serverUrl, + status: 'fail', + errors: [errorMessage], + responseTime, + override: endpointInfo.override, + queryParams: endpointInfo.queryParams + }; + } +} + +export function compareResponses( + baseResponse: any, + betaResponse: any, + tolerance: number = 0.1 +): { isMatch: boolean; differences: string[] } { + const differences: string[] = []; + + function deepCompare(obj1: any, obj2: any, path: string = 'root'): void { + if (typeof obj1 !== typeof obj2) { + differences.push(`${path}: Type mismatch - base: ${typeof obj1}, beta: ${typeof obj2}`); + return; + } + + if (obj1 === null || obj2 === null) { + if (obj1 !== obj2) { + differences.push(`${path}: Null mismatch - base: ${obj1}, beta: ${obj2}`); + } + return; + } + + if (typeof obj1 === 'number' && typeof obj2 === 'number') { + if (obj1 === 0 && obj2 === 0) return; + + const deviation = Math.abs((obj1 - obj2) / (obj1 || 1)); + if (deviation > tolerance) { + differences.push(`${path}: Value deviation ${(deviation * 100).toFixed(2)}% - base: ${obj1}, beta: ${obj2}`); + } + return; + } + + if (typeof obj1 === 'string' || typeof obj1 === 'boolean') { + if (obj1 !== obj2) { + differences.push(`${path}: Value mismatch - base: ${obj1}, beta: ${obj2}`); + } + return; + } + + if (Array.isArray(obj1) && Array.isArray(obj2)) { + if (obj1.length !== obj2.length) { + differences.push(`${path}: Array length mismatch - base: ${obj1.length}, beta: ${obj2.length}`); + } + + const minLength = Math.min(obj1.length, obj2.length); + + if (minLength <= 10) { + for (let i = 0; i < minLength; i++) { + deepCompare(obj1[i], obj2[i], `${path}[${i}]`); + } + } else { + const indicesToCompare = new Set(); + + for (let i = 0; i < 10; i++) { + indicesToCompare.add(i); + } + + for (let i = Math.max(10, minLength - 10); i < minLength; i++) { + indicesToCompare.add(i); + } + + //highest 10 items (sort by numerical value if applicable, or by 'date' property for objects) + try { + const sortedIndices1 = [...Array(obj1.length).keys()].sort((a, b) => { + const val1 = obj1[a]; + const val2 = obj1[b]; + + if (val1 && typeof val1 === 'object' && 'date' in val1 && val2 && typeof val2 === 'object' && 'tvl' in val2) { + return (Number(val2.tvl) || 0) - (Number(val1.tvl) || 0); + } + if (typeof val1 === 'number' && typeof val2 === 'number') { + return val2 - val1; + } + if (val1 && typeof val1 === 'object' && val2 && typeof val2 === 'object') { + const numericKeys = Object.keys(val1).filter(key => typeof val1[key] === 'number'); + if (numericKeys.length > 0) { + const key = numericKeys[0]; + return (Number(val2[key]) || 0) - (Number(val1[key]) || 0); + } + } + return 0; + }); + + for (let i = 0; i < Math.min(10, sortedIndices1.length); i++) { + indicesToCompare.add(sortedIndices1[i]); + } + } catch (error) { + } + + const availableIndices: number[] = []; + for (let i = 0; i < minLength; i++) { + if (!indicesToCompare.has(i)) { + availableIndices.push(i); + } + } + + for (let i = availableIndices.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [availableIndices[i], availableIndices[j]] = [availableIndices[j], availableIndices[i]]; + } + + for (let i = 0; i < Math.min(10, availableIndices.length); i++) { + indicesToCompare.add(availableIndices[i]); + } + + for (const index of Array.from(indicesToCompare).sort((a, b) => a - b)) { + if (index < minLength) { + deepCompare(obj1[index], obj2[index], `${path}[${index}]`); + } + } + } + return; + } + + if (typeof obj1 === 'object' && typeof obj2 === 'object') { + const keys1 = Object.keys(obj1); + const keys2 = Object.keys(obj2); + + const allKeys = [...new Set([...keys1, ...keys2])]; + for (const key of allKeys) { + if (!(key in obj1)) { + differences.push(`${path}.${key}: Missing in base response`); + } else if (!(key in obj2)) { + differences.push(`${path}.${key}: Missing in beta response`); + } else { + deepCompare(obj1[key], obj2[key], `${path}.${key}`); + } + } + } + } + + deepCompare(baseResponse, betaResponse); + + return { + isMatch: differences.length === 0, + differences + }; +} + +export function sanitizeUrlForDisplay(url: string): string { + if (url.includes('pro-api.llama.fi')) { + const parts = url.split('/'); + if (parts.length >= 4 && parts[2] === 'pro-api.llama.fi') { + return parts.slice(0, 3).join('/') + '/' + parts.slice(4).join('/'); + } + } + return url; +} + +export function buildEndpointDisplayPath(endpoint: string, queryParams?: Record): string { + if (!queryParams || Object.keys(queryParams).length === 0) { + return endpoint; + } + + const params = new URLSearchParams(queryParams); + return `${endpoint}?${params.toString()}`; +} + +export function validateBetaConfiguration(): { isValid: boolean; warnings: string[] } { + const warnings: string[] = []; + let isValid = true; + + const requiredBetaUrls = [ + { key: 'BETA_API_URL', domain: 'api.llama.fi' }, + { key: 'BETA_COINS_URL', domain: 'coins.llama.fi' }, + { key: 'BETA_STABLECOINS_URL', domain: 'stablecoins.llama.fi' }, + { key: 'BETA_YIELDS_URL', domain: 'yields.llama.fi' }, + { key: 'BETA_PRO_API_URL', domain: 'pro-api.llama.fi' }, + { key: 'BETA_BRIDGES_URL', domain: 'bridges.llama.fi' } + ]; + + for (const { key, domain } of requiredBetaUrls) { + const url = process.env[key]; + if (!url) { + warnings.push(`${key} not configured - ${domain} endpoints will fallback to BETA_API_URL`); + } else if (!url.startsWith('http')) { + warnings.push(`${key} should start with https:// - got: ${url}`); + isValid = false; + } + } + + if (!process.env.BETA_API_URL) { + warnings.push('BETA_API_URL not configured - beta comparison will fail'); + isValid = false; + } + + return { isValid, warnings }; +} + +export { sendMessage } from '../../src/utils/discord'; \ No newline at end of file diff --git a/defi/src/utils/compareAPI.ts b/defi/src/utils/compareAPI.ts new file mode 100644 index 0000000000..0a9124bdd8 --- /dev/null +++ b/defi/src/utils/compareAPI.ts @@ -0,0 +1,532 @@ +/** + * compareAPI.ts - API Comparison Tool + * + * Compares two API environments (e.g., production vs beta) for deployment validation. + * Tests both schema compliance AND actual data values. + * + * Why data comparison works: Both environments use the same database. + * When called simultaneously, they query the same data snapshot. + * Any differences indicate bugs in the candidate deployment. + * + * Supports both free and pro API testing. + * + * Usage: + * npm run compare-api # Compare free API + * npm run compare-api -- --api-type pro # Compare pro API + * + * See API_VALIDATION_README.md for full documentation. + */ + +import fs from 'fs'; +import path from 'path'; +import dotenv from 'dotenv'; +import { + loadOpenApiSpec, + getAllEndpoints, + fetchWithRetry, + validateResponseAgainstSchema, + compareResponses, + getBetaServerUrl, + EndpointInfo, + sendMessage, +} from './apiValidateUtils'; + +dotenv.config({ path: path.join(__dirname, '../../.env') }); + +// DEBUG: Set specific endpoint to test only that endpoint (for debugging) +// Example: '/api/v2/chains' or '/api/oracles' +// Set to null to test all endpoints +// const DEBUG_ENDPOINT: string | null = null; +const DEBUG_ENDPOINT = '/api/v2/chains'; +// const DEBUG_ENDPOINT = '/api/oracles'; + +interface ComparisonResult { + endpoint: string; + prodUrl: string; + betaUrl: string; + status: 'pass' | 'fail' | 'schema_fail' | 'network_error'; + prodResponseTime?: number; + betaResponseTime?: number; + schemaValidation: { + prod: { valid: boolean; errors: string[] }; + beta: { valid: boolean; errors: string[] }; + }; + dataComparison?: { + isMatch: boolean; + differences: string[]; + }; + errors: string[]; + // TEMP DEBUG: Store actual responses for debugging + prodResponse?: any; + betaResponse?: any; +} + +interface ComparisonReport { + timestamp: string; + apiType: 'free' | 'pro'; + tolerance: number; + summary: { + total: number; + passed: number; + failed: number; + schemaFailures: number; + networkErrors: number; + skippedDataComparison: number; + averageProdResponseTime: number; + averageBetaResponseTime: number; + }; + failedEndpoints: ComparisonResult[]; + significantDifferences: string[]; + passedEndpoints?: ComparisonResult[]; +} + +async function compareEndpoint( + endpoint: EndpointInfo, + tolerance: number = 0.1 +): Promise { + const prodUrl = `${endpoint.serverUrl}${endpoint.path}`; + const betaUrl = getBetaServerUrl(prodUrl); + + const result: ComparisonResult = { + endpoint: endpoint.path, + prodUrl, + betaUrl, + status: 'fail', + schemaValidation: { + prod: { valid: false, errors: [] }, + beta: { valid: false, errors: [] } + }, + errors: [] + }; + + try { + // Fetch from both prod and beta in parallel with 5 retries + const [prodResponse, betaResponse] = await Promise.allSettled([ + (async () => { + const start = Date.now(); + const response = await fetchWithRetry(prodUrl, 5, 1000, 45000); + const responseTime = Date.now() - start; + return { data: response.data, responseTime }; + })(), + (async () => { + const start = Date.now(); + const response = await fetchWithRetry(betaUrl, 5, 1000, 45000); + const responseTime = Date.now() - start; + return { data: response.data, responseTime }; + })() + ]); + + if (prodResponse.status === 'rejected') { + result.errors.push(`Prod API error: ${prodResponse.reason.message}`); + result.status = 'network_error'; + return result; + } + + if (betaResponse.status === 'rejected') { + result.errors.push(`Beta API error: ${betaResponse.reason.message}`); + result.status = 'network_error'; + return result; + } + + const prodData = prodResponse.value.data; + const betaData = betaResponse.value.data; + result.prodResponseTime = prodResponse.value.responseTime; + result.betaResponseTime = betaResponse.value.responseTime; + + // TEMP DEBUG: Store responses in result for JSON report + if (DEBUG_ENDPOINT) { + result.prodResponse = prodData; + result.betaResponse = betaData; + } + + // Validate both responses against OpenAPI schema + const prodValidation = await validateResponseAgainstSchema(prodData, endpoint.schema); + const betaValidation = await validateResponseAgainstSchema(betaData, endpoint.schema); + + result.schemaValidation.prod = prodValidation; + result.schemaValidation.beta = betaValidation; + + if (!prodValidation.valid || !betaValidation.valid) { + result.status = 'schema_fail'; + if (!prodValidation.valid) { + result.errors.push(`Prod schema validation failed: ${prodValidation.errors.join(', ')}`); + } + if (!betaValidation.valid) { + result.errors.push(`Beta schema validation failed: ${betaValidation.errors.join(', ')}`); + } + return result; + } + + // Both schemas are valid - now compare the actual data + // Since prod and beta use the same DB and are called simultaneously, + // they should return the same (or very similar) values + const comparison = compareResponses(prodData, betaData, tolerance); + result.dataComparison = comparison; + + if (comparison.isMatch) { + result.status = 'pass'; + } else { + result.status = 'fail'; + result.errors.push(`Data mismatch: ${comparison.differences.length} differences found`); + } + + return result; + + } catch (error: any) { + result.errors.push(`Unexpected error: ${error.message}`); + result.status = 'network_error'; + return result; + } +} + +async function compareAllEndpoints( + apiType: 'free' | 'pro' = 'free', + tolerance: number = 0.1, + specificEndpoint?: string, + specificDomain?: string, + includeSuccess: boolean = false +): Promise { + + try { + const spec = await loadOpenApiSpec(apiType); + let endpoints = getAllEndpoints(spec, true, apiType); + + // DEBUG: Filter to specific endpoint if DEBUG_ENDPOINT is set + if (DEBUG_ENDPOINT) { + endpoints = endpoints.filter(ep => ep.path === DEBUG_ENDPOINT); + console.log(`[DEBUG MODE] Testing only endpoint: ${DEBUG_ENDPOINT}`); + if (endpoints.length === 0) { + console.log(`[DEBUG MODE] No endpoint found matching: ${DEBUG_ENDPOINT}`); + } + } + + if (specificEndpoint) { + endpoints = endpoints.filter(ep => ep.path.includes(specificEndpoint)); + console.log(`filtering to endpoints containing: ${specificEndpoint}`); + } + + if (specificDomain) { + endpoints = endpoints.filter(ep => ep.serverUrl.includes(specificDomain)); + console.log(`filtering to domain: ${specificDomain}`); + } + + console.log(`comparing ${endpoints.length} endpoints between prod and beta...`); + + const results: ComparisonResult[] = []; + const prodResponseTimes: number[] = []; + const betaResponseTimes: number[] = []; + + for (const [index, endpoint] of endpoints.entries()) { + const progress = ((index + 1) / endpoints.length * 100).toFixed(1); + process.stdout.write(`\rprogress: ${progress}% (${index + 1}/${endpoints.length}) - comparing ${endpoint.path}`); + + try { + const result = await compareEndpoint(endpoint, tolerance); + results.push(result); + + if (result.prodResponseTime) prodResponseTimes.push(result.prodResponseTime); + if (result.betaResponseTime) betaResponseTimes.push(result.betaResponseTime); + + if (result.status === 'fail') { + console.log(`\nāŒ DATA MISMATCH: ${endpoint.path}`); + result.errors.forEach(error => console.log(` • ${error}`)); + + if (result.dataComparison && result.dataComparison.differences.length > 0) { + const topDiffs = result.dataComparison.differences.slice(0, 3); + topDiffs.forEach(diff => console.log(` šŸ“Š ${diff}`)); + if (result.dataComparison.differences.length > 3) { + console.log(` ... and ${result.dataComparison.differences.length - 3} more differences`); + } + } + } else if (result.status === 'schema_fail') { + console.log(`\nšŸ”§ SCHEMA FAIL: ${endpoint.path}`); + result.errors.forEach(error => console.log(` • ${error}`)); + } + + await new Promise(resolve => setTimeout(resolve, 200)); + + } catch (error: any) { + const failedResult: ComparisonResult = { + endpoint: endpoint.path, + prodUrl: `${endpoint.serverUrl}${endpoint.path}`, + betaUrl: getBetaServerUrl(`${endpoint.serverUrl}${endpoint.path}`), + status: 'network_error', + schemaValidation: { + prod: { valid: false, errors: [] }, + beta: { valid: false, errors: [] } + }, + errors: [`Critical error: ${error.message}`] + }; + results.push(failedResult); + console.log(`\nERROR: ${endpoint.path} - ${error.message}`); + } + } + + console.log('\n'); + + const passed = results.filter(r => r.status === 'pass').length; + const failed = results.filter(r => r.status === 'fail').length; + const schemaFailures = results.filter(r => r.status === 'schema_fail').length; + const networkErrors = results.filter(r => r.status === 'network_error').length; + const skippedDataComparison = 0; // We compare data for all endpoints + + const avgProdResponseTime = prodResponseTimes.length > 0 + ? Math.round(prodResponseTimes.reduce((a, b) => a + b, 0) / prodResponseTimes.length) + : 0; + const avgBetaResponseTime = betaResponseTimes.length > 0 + ? Math.round(betaResponseTimes.reduce((a, b) => a + b, 0) / betaResponseTimes.length) + : 0; + + const significantDifferences: string[] = []; + results.forEach(result => { + if (result.dataComparison && !result.dataComparison.isMatch) { + result.dataComparison.differences.forEach(diff => { + if (!significantDifferences.includes(diff)) { + significantDifferences.push(diff); + } + }); + } + }); + + const failedResults = results.filter(r => r.status !== 'pass'); + const passedResults = results.filter(r => r.status === 'pass'); + + const report: ComparisonReport = { + timestamp: new Date().toISOString(), + apiType, + tolerance, + summary: { + total: results.length, + passed, + failed, + schemaFailures, + networkErrors, + skippedDataComparison, + averageProdResponseTime: avgProdResponseTime, + averageBetaResponseTime: avgBetaResponseTime + }, + failedEndpoints: failedResults, + significantDifferences: significantDifferences.slice(0, 20) + }; + + // Only include successful results if requested + if (includeSuccess) { + report.passedEndpoints = passedResults; + } + + return report; + + } catch (error: any) { + console.error('error during comparison:', error.message); + throw error; + } +} + +async function generateComparisonReport(report: ComparisonReport, outputFile?: string): Promise { + const { summary, significantDifferences, apiType, tolerance } = report; + + console.log('\ncomparison summary'); + console.log('='.repeat(50)); + console.log(`api: ${apiType}`); + console.log(`total endpoints: ${summary.total}`); + console.log(`passed (schema + data): ${summary.passed}`); + console.log(`data mismatch: ${summary.failed}`); + console.log(`schema failed: ${summary.schemaFailures}`); + console.log(`network errors: ${summary.networkErrors}`); + console.log(`success rate: ${((summary.passed / summary.total) * 100).toFixed(1)}%`); + console.log(`prod avg resp: ${summary.averageProdResponseTime}ms`); + console.log(`beta avg resp: ${summary.averageBetaResponseTime}ms`); + console.log(`tolerance: ${tolerance * 100}%`); + + if (significantDifferences.length > 0) { + console.log('\nTop differences found:'); + significantDifferences.slice(0, 10).forEach((diff, index) => { + console.log(`${index + 1}. ${diff}`); + }); + + if (significantDifferences.length > 10) { + console.log(`... and ${significantDifferences.length - 10} more difference patterns`); + } + } + + const reportFile = outputFile || path.join(__dirname, '../beta_comparison_report.json'); + fs.writeFileSync(reportFile, JSON.stringify(report, null, 2)); + console.log(`\nfull comparison report saved to: ${reportFile}`); +} + +async function sendNotification(report: ComparisonReport, isDryRun: boolean = false): Promise { + const { summary, apiType } = report; + + if (summary.failed === 0 && summary.schemaFailures === 0 && summary.networkErrors === 0) { + console.log('\nall comparisons passed - no notification needed'); + return; + } + + const failedEndpointsList = report.failedEndpoints.map(r => `${r.endpoint} (${r.status})`); + + const failureMessage = "```" + + `beta comparison - issues detected\n\n` + + `${apiType} api:\n` + + `• total endpoints: ${summary.total}\n` + + `• passed: ${summary.passed}\n` + + `• data mismatch: ${summary.failed}\n` + + `• schema failed: ${summary.schemaFailures}\n` + + `• network errors: ${summary.networkErrors}\n` + + `• success rate: ${((summary.passed / summary.total) * 100).toFixed(1)}%\n` + + `• prod avg resp: ${summary.averageProdResponseTime}ms\n` + + `• beta avg resp: ${summary.averageBetaResponseTime}ms\n\n` + + `failing endpoints (${Math.min(failedEndpointsList.length, 20)} of ${failedEndpointsList.length}):\n` + + failedEndpointsList.slice(0, 20).map((endpoint, i) => `${i + 1}. ${endpoint}`).join('\n') + + (failedEndpointsList.length > 20 ? `\n... and ${failedEndpointsList.length - 20} more` : '') + + `\n\n${report.timestamp}` + + "```"; + + console.log('\nsending discord notification'); + + if (!isDryRun) { + await sendMessage(failureMessage, process.env.DISCORD_WEBHOOK_URL, false); + } else { + console.log('would send:\n', failureMessage); + } +} + +async function main(): Promise { + const args = process.argv.slice(2); + + let apiType: 'free' | 'pro' = 'free'; + let tolerance = 0.1; + let specificEndpoint: string | undefined; + let specificDomain: string | undefined; + let outputFile: string | undefined; + let isDryRun = false; + let includeSuccess = false; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case '--api-type': + apiType = args[++i] as 'free' | 'pro'; + break; + case '--tolerance': + tolerance = parseFloat(args[++i]); + break; + case '--endpoint': + specificEndpoint = args[++i]; + break; + case '--domain': + specificDomain = args[++i]; + break; + case '--output': + outputFile = args[++i]; + break; + case '--dry-run': + isDryRun = true; + break; + case '--include-success': + includeSuccess = true; + break; + case '--help': + console.log(` +╔═══════════════════════════════════════════════════════════════════════════╗ +ā•‘ compareAPI.ts - API Comparison Tool ā•‘ +ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā• + +PURPOSE: + Compares production vs beta API responses for deployment validation. + Tests: Schema compliance + Actual data values. + + Why it works: Prod and beta share the same database. When called + simultaneously, they query the same data. Any differences = bugs in beta! + +USAGE: + npx ts-node src/utils/compareBeta.ts [options] + +OPTIONS: + --api-type API type to compare (default: free) + --tolerance Tolerance for numerical differences (default: 0.1 = 10%) + --endpoint Filter endpoints containing this pattern + --domain Filter to specific domain (e.g., stablecoins.llama.fi) + --output Custom output file for JSON report + --dry-run Skip Discord notifications + --include-success Include successful endpoints in JSON report (default: false) + --help Show this help message + +TOLERANCE EXPLAINED: + 0.0 = Exact match required + 0.05 = 5% difference allowed + 0.1 = 10% difference allowed (default) + 0.2 = 20% difference allowed + +EXAMPLES: + # Compare all free API endpoints (prod vs beta) + npm run compare-api + + # Compare pro API with strict tolerance + npm run compare-api -- --api-type pro --tolerance 0.05 + + # Test specific protocol endpoints + npm run compare-api -- --endpoint /protocol + + # Test stablecoins domain only + npm run compare-api -- --domain stablecoins.llama.fi + + # CI/CD: Run without Discord notification + npm run compare-api -- --dry-run + +OUTPUT: + āœ… Pass - Schema valid + data matches (within tolerance) + āŒ Data mismatch - Values differ between prod and beta + šŸ”§ Schema fail - Response doesn't match OpenAPI schema + āš ļø Network error - Unable to reach endpoint + +TYPICAL WORKFLOW: + 1. Deploy new code to beta environment + 2. Run: npm run compare-api + 3. Review differences (if any) + 4. Fix bugs in beta + 5. Repeat until all tests pass + 6. Promote beta to production + +See API_VALIDATION_README.md for full documentation. + `); + process.exit(0); + } + } + + console.log('api compare tool\n'); + + try { + const report = await compareAllEndpoints(apiType, tolerance, specificEndpoint, specificDomain, includeSuccess); + await generateComparisonReport(report, outputFile); + + const hasIssues = report.summary.failed > 0 || report.summary.schemaFailures > 0 || report.summary.networkErrors > 0; + + if (hasIssues) { + console.log(`\ncomparison completed with issues`); + await sendNotification(report, isDryRun); + process.exit(1); + } else { + console.log('\nall comparisons passed successfully!'); + process.exit(0); + } + + } catch (error: any) { + console.error(`\nerror: ${error.message}`); + + if (!isDryRun) { + await sendMessage( + `beta comparison critical error!\n\n` + + `error: ${error.message}\n` + + `timestamp: ${new Date().toISOString()}`, + process.env.DISCORD_WEBHOOK_URL, + false + ); + } + + process.exit(1); + } +} + +if (require.main === module) { + main(); +} \ No newline at end of file diff --git a/defi/src/utils/validateAPI.ts b/defi/src/utils/validateAPI.ts new file mode 100644 index 0000000000..01019408b0 --- /dev/null +++ b/defi/src/utils/validateAPI.ts @@ -0,0 +1,344 @@ +/** + * validateAPI.ts - OpenAPI Compliance Validator + * + * Validates that API responses match the OpenAPI specification. + * Tests schema structure, required fields, and data types. + * Does NOT compare actual values (use compareAPI.ts for that). + * + * Usage: + * npx ts-node src/utils/validateAPI.ts [--api-type free|pro] [--endpoint pattern] + * + */ + +import fs from 'fs'; +import path from 'path'; +import dotenv from 'dotenv'; +import { + loadOpenApiSpec, + getAllEndpoints, + testEndpoint, + ValidationResult, + sendMessage, + sanitizeUrlForDisplay, + buildEndpointDisplayPath +} from './apiValidateUtils'; + +dotenv.config({ path: path.join(__dirname, '../../.env') }); + +// DEBUG: Set specific endpoint to test only that endpoint (for debugging) +// Example: '/api/v2/chains' or '/api/oracles' +// Set to null to test all endpoints +const DEBUG_ENDPOINT: string | null = null; +// const DEBUG_ENDPOINT = 'protocol/aave'; +// const DEBUG_ENDPOINT = '/api/oracles'; + +interface ValidationReport { + timestamp: string; + environment: string; + summary: { + total: number; + passed: number; + failed: number; + averageResponseTime: number; + }; + failedEndpoints: Array<{ + endpoint: string; + status: string; + errors: string[]; + responseTime?: number; + }>; +} + +async function validateAllEndpoints( + specificEndpoint?: string, + specificDomain?: string, + useBeta: boolean = false +): Promise { + const envLabel = useBeta ? 'BETA' : 'PRODUCTION'; + console.log(`\n${'='.repeat(60)}`); + console.log(`API Validation - ${envLabel}`); + console.log('='.repeat(60)); + + try { + // Load both free and pro specs and merge endpoints + const freeSpec = await loadOpenApiSpec('free'); + const proSpec = await loadOpenApiSpec('pro'); + let endpoints = [ + ...getAllEndpoints(freeSpec, true, 'free'), + ...getAllEndpoints(proSpec, true, 'pro') + ]; + + console.log(`Loaded ${endpoints.length} total endpoints (free + pro)`); + + // DEBUG: Filter to specific endpoint if DEBUG_ENDPOINT is set + if (DEBUG_ENDPOINT) { + endpoints = endpoints.filter(ep => ep.path === DEBUG_ENDPOINT); + console.log(`[DEBUG MODE] Testing only endpoint: ${DEBUG_ENDPOINT}`); + if (endpoints.length === 0) { + console.log(`[DEBUG MODE] No endpoint found matching: ${DEBUG_ENDPOINT}`); + } + } + + if (specificEndpoint) { + endpoints = endpoints.filter(ep => ep.path.includes(specificEndpoint)); + console.log(`Filtering to endpoints containing: ${specificEndpoint}`); + } + + if (specificDomain) { + endpoints = endpoints.filter(ep => ep.serverUrl.includes(specificDomain)); + console.log(`Filtering to domain: ${specificDomain}`); + } + + // If using beta, map URLs to beta environment + if (useBeta) { + const { getBetaServerUrl } = require('./apiValidateUtils'); + endpoints = endpoints.map(ep => ({ + ...ep, + serverUrl: getBetaServerUrl(`${ep.serverUrl}${ep.path}`).replace(ep.path, '') + })); + } + + console.log(`Testing ${endpoints.length} endpoints...\n`); + + const results: ValidationResult[] = []; + const totalResponseTimes: number[] = []; + + // Use defaults: retryCount=5, retryDelay=1000, requestTimeout=45000, requestDelay=500 + const retryCount = 5; + const retryDelay = 1000; + const requestTimeout = 45000; + const requestDelay = 500; + + for (const [index, endpoint] of endpoints.entries()) { + const progress = ((index + 1) / endpoints.length * 100).toFixed(1); + process.stdout.write(`\rProgress: ${progress}% (${index + 1}/${endpoints.length}) - Testing ${endpoint.path}`); + + try { + const result = await testEndpoint(endpoint, false, retryCount, retryDelay, requestTimeout); + results.push(result); + + if (result.responseTime) { + totalResponseTimes.push(result.responseTime); + } + + if (result.status === 'fail') { + const displayPath = buildEndpointDisplayPath(result.endpoint, result.queryParams); + // console.log(`\nFailed: ${displayPath} on ${sanitizeUrlForDisplay(endpoint.serverUrl)}`); + // result.errors.forEach(error => console.log(` • ${error}`)); + } + + await new Promise(resolve => setTimeout(resolve, requestDelay)); + + } catch (error: any) { + const failedResult: ValidationResult = { + endpoint: endpoint.path, + serverUrl: endpoint.serverUrl, + status: 'fail', + errors: [`Unexpected error: ${error.message}`], + queryParams: endpoint.queryParams + }; + results.push(failedResult); + const displayPath = buildEndpointDisplayPath(endpoint.path, endpoint.queryParams); + console.log(`\nError: ${displayPath} - ${error.message}`); + } + } + + console.log('\n'); + + const passed = results.filter(r => r.status === 'pass').length; + const failed = results.filter(r => r.status === 'fail').length; + const averageResponseTime = totalResponseTimes.length > 0 + ? Math.round(totalResponseTimes.reduce((a, b) => a + b, 0) / totalResponseTimes.length) + : 0; + + const failedResults = results.filter(r => r.status === 'fail'); + + const report: ValidationReport = { + timestamp: new Date().toISOString(), + environment: envLabel, + summary: { + total: results.length, + passed, + failed, + averageResponseTime + }, + failedEndpoints: failedResults.map(r => ({ + endpoint: `${r.serverUrl}${buildEndpointDisplayPath(r.endpoint, r.queryParams)}`, + status: r.status, + errors: r.errors, + responseTime: r.responseTime + })) + }; + + return report; + + } catch (error: any) { + console.error('Error during validation:', error.message); + throw error; + } +} + +async function generateReport( + report: ValidationReport, + outputFile?: string +): Promise { + const { summary, failedEndpoints, environment } = report; + + console.log(`\n${'='.repeat(60)}`); + console.log(`VALIDATION SUMMARY - ${environment}`); + console.log('='.repeat(60)); + + console.log(`\nTotal Endpoints: ${summary.total}`); + console.log(`Passed: ${summary.passed}`); + console.log(`Failed: ${summary.failed}`); + console.log(`Avg Response Time: ${summary.averageResponseTime}ms`); + console.log(`Success Rate: ${((summary.passed / summary.total) * 100).toFixed(1)}%`); + + // Show failed endpoints + if (failedEndpoints.length > 0) { + console.log(`\nāŒ FAILED ENDPOINTS (${failedEndpoints.length}):`); + failedEndpoints.forEach((item, index) => { + console.log(` ${index + 1}. ${sanitizeUrlForDisplay(item.endpoint)}`); + item.errors.forEach(error => console.log(` • ${error}`)); + }); + } + + const reportFile = outputFile || path.join(__dirname, '../validation_report.json'); + fs.writeFileSync(reportFile, JSON.stringify(report, null, 2)); + console.log(`\nšŸ“„ Full report saved to: ${reportFile}`); +} + +async function sendNotification(report: ValidationReport): Promise { + const { summary, failedEndpoints, environment } = report; + + if (summary.failed === 0) { + console.log('\nāœ… All validations passed - no notification needed'); + return; + } + + const failureMessage = "```" + + `API Validation ${environment} - Issues Detected\n\n` + + `Total Endpoints: ${summary.total}\n` + + `Passed: ${summary.passed}\n` + + `Failed: ${summary.failed}\n` + + `Success Rate: ${((summary.passed / summary.total) * 100).toFixed(1)}%\n` + + `Avg Response: ${summary.averageResponseTime}ms\n\n` + + `Failed Endpoints (${Math.min(failedEndpoints.length, 20)} of ${failedEndpoints.length}):\n` + + failedEndpoints.slice(0, 20).map((item, i) => + `${i + 1}. ${sanitizeUrlForDisplay(item.endpoint)}` + ).join('\n') + + (failedEndpoints.length > 20 ? `\n... and ${failedEndpoints.length - 20} more` : '') + + `\n\n${report.timestamp}` + + "```"; + + console.log('\nšŸ“¤ Sending Discord notification...'); + + await sendMessage(failureMessage, process.env.DISCORD_WEBHOOK_URL, false); +} + +async function main(): Promise { + const args = process.argv.slice(2); + + let specificEndpoint: string | undefined; + let specificDomain: string | undefined; + let outputFile: string | undefined; + let useBeta = false; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case '--endpoint': + specificEndpoint = args[++i]; + break; + case '--domain': + specificDomain = args[++i]; + break; + case '--output': + outputFile = args[++i]; + break; + case '--beta': + useBeta = true; + break; + case '--help': + console.log(` +╔═══════════════════════════════════════════════════════════════════════════╗ +ā•‘ validateAPI.ts - OpenAPI Validator ā•‘ +ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā• + +PURPOSE: + Validates API responses match the OpenAPI specification. + Merges and tests ALL endpoints (free + pro) in a single run. + Tests: Schema structure, required fields, data types. + + Defaults: 5 retries, 1-5s backoff, 45s timeout, 500ms request delay + +USAGE: + npm run validate-api # Validate production (default) + npm run validate-api-beta # Validate beta environment + +OPTIONS: + --beta Test beta URLs instead of production + --endpoint Filter endpoints containing this pattern + --domain Filter to specific domain (e.g., coins.llama.fi) + --output Custom output file for JSON report + --help Show this help message + +EXAMPLES: + # Validate production (all endpoints) + npm run validate-api + + # Validate beta (all endpoints) + npm run validate-api-beta + + # Test specific endpoint + npm run validate-api -- --endpoint /protocol/aave + + # Test coins domain only + npm run validate-api -- --domain coins.llama.fi + +OUTPUT: + All endpoints tested together (no free/pro separation) + āœ… Pass - Response matches OpenAPI schema + āŒ Fail - Response doesn't match schema + + Report: defi/src/validation_report.json + +See API_VALIDATION_README.md for full documentation. + `); + process.exit(0); + } + } + + console.log('šŸ” API Validation Tool\n'); + + try { + const report = await validateAllEndpoints(specificEndpoint, specificDomain, useBeta); + await generateReport(report, outputFile); + + if (report.summary.failed > 0) { + console.log(`\nāŒ Validation failed with ${report.summary.failed} endpoint failures`); + await sendNotification(report); + process.exit(1); + } else { + console.log('\nāœ… All validations passed!'); + process.exit(0); + } + + } catch (error: any) { + console.error(`\nšŸ’„ Critical error: ${error.message}`); + + await sendMessage( + `API Validation Critical Error!\n\n` + + `Error: ${error.message}\n` + + `Timestamp: ${new Date().toISOString()}`, + process.env.DISCORD_WEBHOOK_URL, + false + ); + + process.exit(1); + } +} + +if (require.main === module) { + main(); +}