diff --git a/.env.example b/.env.example index cfca3e9..a18ce9b 100644 --- a/.env.example +++ b/.env.example @@ -5,4 +5,13 @@ HELIUS_RATE_LIMIT=50 # BirdEye API Configuration BIRDEYE_API_KEY=your_birdeye_api_key_here # BIRDEYE_RATE_LIMIT: 1 RPS default -BIRDEYE_RATE_LIMIT=1 \ No newline at end of file +BIRDEYE_RATE_LIMIT=1 + +# BirdEye Price History Timeframe +# Controls granularity of historical price data for ATH detection +# Options: 1m, 5m, 15m, 30m, 1H, 4H, 1D +# - 1D (default): Best API efficiency, ~1 API call per year of data, daily precision +# - 1H: More precise (hourly), but ~9 API calls per year of data +# - 1m: Highest precision (minute-level), but ~525 API calls per year (not recommended) +# Recommendation: Use '1D' for best performance unless you need intraday precision +BIRDEYE_PRICE_TIMEFRAME=1D \ No newline at end of file diff --git a/src/config/index.ts b/src/config/index.ts index 5eb8557..a123a05 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -8,8 +8,17 @@ export interface SendoAnalyserConfig { birdeyeApiKey?: string; birdeyeRateLimit?: number; // requests per second heliusRateLimit?: number; // requests per second + birdeyePriceTimeframe?: BirdEyeTimeframe; // historical price data granularity } +/** + * BirdEye API Timeframe options for historical price data + * - 1m, 5m, 15m, 30m: High precision but requires many API calls for long periods + * - 1H, 4H: Balanced precision and API efficiency + * - 1D: Best API efficiency (1000 days per request) - RECOMMENDED for ATH detection + */ +export type BirdEyeTimeframe = '1m' | '5m' | '15m' | '30m' | '1H' | '4H' | '1D'; + /** * Default configuration values */ @@ -21,6 +30,9 @@ export const SENDO_ANALYSER_DEFAULTS = { API_USAGE_PERCENT: 80, // Use 80% of available RPS for safety margin BIRDEYE_API_BASE: 'https://public-api.birdeye.so/defi', HELIUS_NETWORK: 'mainnet' as const, + // Price history timeframe: '1D' = daily candles (best for ATH, ~1 API call per year of data) + // Alternative: '1H' = hourly (more precise but ~9 API calls per year) + BIRDEYE_PRICE_TIMEFRAME: '1D' as BirdEyeTimeframe, }; /** @@ -39,12 +51,14 @@ export function getSendoAnalyserConfig(runtime: IAgentRuntime): SendoAnalyserCon const birdeyeApiKey = runtime.getSetting('BIRDEYE_API_KEY') as string; const birdeyeRateLimit = parseInt(runtime.getSetting('BIRDEYE_RATE_LIMIT') as string || String(SENDO_ANALYSER_DEFAULTS.BIRDEYE_RATE_LIMIT)); const heliusRateLimit = parseInt(runtime.getSetting('HELIUS_RATE_LIMIT') as string || String(SENDO_ANALYSER_DEFAULTS.HELIUS_RATE_LIMIT)); + const birdeyePriceTimeframe = (runtime.getSetting('BIRDEYE_PRICE_TIMEFRAME') as BirdEyeTimeframe) || SENDO_ANALYSER_DEFAULTS.BIRDEYE_PRICE_TIMEFRAME; return { heliusApiKey, birdeyeApiKey: birdeyeApiKey || undefined, birdeyeRateLimit: birdeyeRateLimit || SENDO_ANALYSER_DEFAULTS.BIRDEYE_RATE_LIMIT, heliusRateLimit: heliusRateLimit || SENDO_ANALYSER_DEFAULTS.HELIUS_RATE_LIMIT, + birdeyePriceTimeframe: birdeyePriceTimeframe || SENDO_ANALYSER_DEFAULTS.BIRDEYE_PRICE_TIMEFRAME, }; } diff --git a/src/services/api/birdeyes.ts b/src/services/api/birdeyes.ts index 377cdcb..0494ad5 100644 --- a/src/services/api/birdeyes.ts +++ b/src/services/api/birdeyes.ts @@ -44,15 +44,14 @@ export interface BirdeyeService { getPriceAnalysis( mint: string, - purchaseTimestamp: number + purchaseTimestamp: number, + timeframe?: '1m' | '5m' | '15m' | '30m' | '1H' | '4H' | '1D' ): Promise<{ purchasePrice: number; currentPrice: number; athPrice: number; athTimestamp: number; priceHistory: BirdEyePriceData[]; - symbol: string | null; - name: string | null; } | null>; } @@ -206,50 +205,21 @@ export function getBirdeyeService( }); }; - /** - * Get token metadata (symbol, name) from BirdEye - */ - const getTokenMetadata = async (mint: string) => { - return dynamicLimiter.schedule(async () => { - try { - const response = await axios.get(`https://public-api.birdeye.so/defi/token_overview`, { - params: { address: mint }, - headers: { - 'accept': 'application/json', - 'x-chain': 'solana', - ...(apiKey && { 'X-API-KEY': apiKey }) - }, - timeout: 30000 - }); - - if (response.data.success && response.data.data) { - return { - symbol: response.data.data.symbol || null, - name: response.data.data.name || null - }; - } - return { symbol: null, name: null }; - } catch (error) { - console.error(`BirdEye metadata error for ${mint}:`, axios.isAxiosError(error) ? `${error.response?.status} - ${error.message}` : (error instanceof Error ? error.message : String(error))); - return { symbol: null, name: null }; - } - }); - }; - /** * Analyze price from purchase to now and compute ATH. + * Note: Metadata (symbol, name) should now be fetched via Helius getTokenMetadataBatch + * for better performance (batch request vs individual calls) */ const getPriceAnalysis = async ( mint: string, - purchaseTimestamp: number + purchaseTimestamp: number, + timeframe?: '1m' | '5m' | '15m' | '30m' | '1H' | '4H' | '1D' ) => { const now = Math.floor(Date.now() / 1000); - // Fetch metadata and price history in parallel - const [metadata, priceHistory] = await Promise.all([ - getTokenMetadata(mint), - getFullHistoricalPrices(mint, purchaseTimestamp, now, '1H') - ]); + // Fetch only price history (metadata is now fetched via Helius batch API) + // Default to '1D' for best API efficiency (1000 days per request) + const priceHistory = await getFullHistoricalPrices(mint, purchaseTimestamp, now, timeframe || '1D'); if (!priceHistory.length) return null; @@ -271,9 +241,7 @@ export function getBirdeyeService( currentPrice, athPrice, athTimestamp, - priceHistory, - symbol: metadata.symbol, - name: metadata.name + priceHistory }; }; diff --git a/src/services/api/helius.ts b/src/services/api/helius.ts index 5015ea8..94eeee0 100644 --- a/src/services/api/helius.ts +++ b/src/services/api/helius.ts @@ -15,6 +15,7 @@ export interface HeliusService { paginationToken?: string; hasMore: boolean; }>; + getTokenMetadataBatch(mints: string[]): Promise>; } /** @@ -133,5 +134,54 @@ export function createHeliusService(apiKey: string, requestsPerSecond: number = }; }); }, + + getTokenMetadataBatch: async (mints: string[]) => { + // Use Helius DAS API getAssetBatch to fetch metadata for multiple tokens + // This is MUCH more efficient than calling Birdeye token_overview for each token + // Limit: up to 1000 tokens per request + + return withRateLimit(async () => { + const metadataMap = new Map(); + + // Process in batches of 1000 (API limit) + for (let i = 0; i < mints.length; i += 1000) { + const batch = mints.slice(i, i + 1000); + + try { + const response = await fetch(`https://mainnet.helius-rpc.com/?api-key=${apiKey}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'getAssetBatch', + params: { + ids: batch + } + }) + }); + + const { result } = await response.json(); + + if (result && Array.isArray(result)) { + result.forEach((asset: any) => { + if (asset && asset.id) { + // Extract symbol and name from metadata + const symbol = asset.content?.metadata?.symbol || asset.content?.json_uri?.symbol || null; + const name = asset.content?.metadata?.name || asset.content?.json_uri?.name || null; + + metadataMap.set(asset.id, { symbol, name }); + } + }); + } + } catch (error) { + console.error(`[Helius] getAssetBatch error for batch ${i}-${i + batch.length}:`, error); + // Continue to next batch even if this one fails + } + } + + return metadataMap; + }); + }, }; } diff --git a/src/services/workers/analysisWorker.ts b/src/services/workers/analysisWorker.ts index 16d4046..ffd22e5 100644 --- a/src/services/workers/analysisWorker.ts +++ b/src/services/workers/analysisWorker.ts @@ -5,6 +5,7 @@ import { getTransactionsWithCache } from '../cache/transactionCache'; import { processNewTransactions } from './workerHelpers'; import { serializeBigInt } from '../../utils/serializeBigInt'; import { upsertTokenResults, getTokenResults } from '../analysis/tokenResults'; +import { SENDO_ANALYSER_DEFAULTS } from '../../config/index.js'; import type { HeliusService } from '../api/helius'; import type { BirdeyeService } from '../api/birdeyes'; @@ -186,7 +187,21 @@ export async function processAnalysisJobAsync( }); }); - // Fetch price analysis for all unique mints in parallel with per-token timeout + // Step 1: Batch-fetch metadata with Helius (MUCH more efficient than Birdeye!) + // Extract unique mints from all trades + const uniqueMints = Array.from(new Set(Array.from(mintsToAnalyze.values()).map(({ mint }) => mint))); + logger.info(`[ProcessAnalysisJob] Batch-fetching metadata for ${uniqueMints.length} tokens via Helius...`); + + let metadataMap = new Map(); + try { + metadataMap = await heliusService.getTokenMetadataBatch(uniqueMints); + logger.info(`[ProcessAnalysisJob] Successfully fetched metadata for ${metadataMap.size}/${uniqueMints.length} tokens`); + } catch (error: any) { + logger.warn(`[ProcessAnalysisJob] Failed to batch-fetch metadata:`, error?.message || String(error)); + // Continue without metadata - will use fallback + } + + // Step 2: Fetch price history from Birdeye (prices only, no metadata) // Use Promise.allSettled to handle individual failures gracefully // Calculate dynamic timeout based on: // - Number of active jobs (fair sharing) @@ -200,11 +215,13 @@ export async function processAnalysisJobAsync( const priceAnalysesPromises = Array.from(mintsToAnalyze.values()).map(async ({ mint, timestamp }) => { try { // Use dynamic timeout that adapts to current load and batch size + // Use configured timeframe from SENDO_ANALYSER_DEFAULTS (default: '1D' for best API efficiency) const analysis = await withTimeout( - cachedBirdeyeService.getPriceAnalysis(mint, timestamp), + cachedBirdeyeService.getPriceAnalysis(mint, timestamp, SENDO_ANALYSER_DEFAULTS.BIRDEYE_PRICE_TIMEFRAME), dynamicTimeout, `Price timeout for ${mint.slice(0, 8)}` ); + return { mint, timestamp, analysis }; } catch (error: any) { logger.warn(`[ProcessAnalysisJob] Skipping price for ${mint.slice(0, 8)}... (${error?.message || String(error)})`); @@ -281,8 +298,11 @@ export async function processAnalysisJobAsync( let missedATH = 0; let gainLoss = 0; let pnl = 0; - let symbol = trade.tokenSymbol; - let name = null; + + // Get metadata from Helius (batch-fetched earlier) + const metadata = metadataMap.get(trade.mint); + let symbol = metadata?.symbol || trade.tokenSymbol; + let name = metadata?.name || null; if (!priceAnalysis) { tradesMissingPrice++; @@ -290,8 +310,6 @@ export async function processAnalysisJobAsync( if (priceAnalysis) { const { purchasePrice, currentPrice, athPrice } = priceAnalysis; - symbol = priceAnalysis.symbol || trade.tokenSymbol; - name = priceAnalysis.name; // Store full token name // Calculate USD volume for this trade volume = Number(trade.amount) * Number(purchasePrice); diff --git a/src/utils/parseTrade.ts b/src/utils/parseTrade.ts index d1f6624..d6f1f68 100644 --- a/src/utils/parseTrade.ts +++ b/src/utils/parseTrade.ts @@ -180,7 +180,7 @@ export const parseTransactionsWithPriceAnalysis = async (transactions: any[], bi const currentPrice = priceAnalysis.currentPrice; const athPrice = priceAnalysis.athPrice; - tradeData.tokenSymbol = priceAnalysis.symbol || undefined; + // Note: tokenSymbol should be enriched separately via Helius metadata tradeData.volume = tokenAmount * purchasePrice; // Volume in USD tradeData.missedATH = athPrice > 0 ? ((athPrice - currentPrice) / athPrice) * 100 : 0; tradeData.gainLoss = purchasePrice > 0 ? ((currentPrice - purchasePrice) / purchasePrice) * 100 : 0;