Coins Api

src/api2.ts

@@ -2,4 +2,4 @@ export * as util from "./util";
 export * as eth from "./eth";
 export * as erc20 from "./erc20";
 export * as abi from "./abi/abi2";
-export * as config from "./general";
+export * as config from "./general";

src/index.test.ts

@@ -116,6 +116,10 @@ test("imports", async () => {
         "writeCache": [Function],
         "writeExpiringJsonCache": [Function],
       },
+      "coins": {
+        "getMcaps": [Function],
+        "getPrices": [Function],
+      },
       "elastic": {
         "addDebugLog": [Function],
         "addErrorLog": [Function],

src/index.ts

@@ -17,6 +17,7 @@ export * as indexer from "./util/indexer";
 export * as types from "./types";
 export * as tron from "./abi/tron";
 export * as erc20 from "./erc20";
+export * as coins from "./util/coinsApi";
 
 export const log = debugLog
 export const logTable = debugTable

src/util/coinsApi.test.ts

@@ -0,0 +1,18 @@
+import { getPrices, getMcaps } from "./coinsApi";
+
+test("coinsApi - mcaps", async () => {
+    const res = await getMcaps(["coingecko:tether"], "now");
+    expect(res["coingecko:tether"].mcap).toBeGreaterThan(100_000);
+    expect(res["coingecko:tether"].mcap).toBeLessThan(1_000_000_000_000);
+    expect(res["coingecko:tether"].timestamp).toBeGreaterThan(Math.floor(Date.now() / 1e3 - 3600));
+    expect(Object.keys(res).length).toBe(1);
+})
+
+test("coinsApi - prices", async () => {
+    const res = await getPrices(["coingecko:tether", "ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7", "solana:So11111111111111111111111111111111111111112"], "now");
+    expect(res["coingecko:tether"].price).toBe(1);
+    expect(res["ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7"].symbol).toBe("USDT");
+    expect(res["ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7"].decimals).toBe(6);
+    expect(Object.keys(res).length).toBe(3);
+    expect(res["solana:So11111111111111111111111111111111111111112"].timestamp).toBeGreaterThan(Math.floor(Date.now() / 1e3 - 3600));
+})

src/util/coinsApi.ts

@@ -0,0 +1,172 @@
+import axios from "axios";
+import { getEnvValue } from "./env";
+import runInPromisePool from "./promisePool";
+
+type CoinsApiData = {
+  decimals: number;
+  price: number;
+  symbol: string;
+  timestamp: number;
+  PK?: string;
+};
+
+type McapsApiData = {
+  mcap: number;
+  timestamp: number;
+};
+
+const coinsApiKey = getEnvValue("COINS_API_KEY")
+const bodySize = 2; // 100;
+
+function getBodies(readKeys: string[], timestamp: number | "now") {
+  const bodies: string[] = [];
+  for (let i = 0; i < readKeys.length; i += bodySize) {
+    const body = {
+      coins: readKeys.slice(i, i + bodySize),
+    } as any;
+    if (timestamp !== "now") body.timestamp = timestamp;
+    bodies.push(JSON.stringify(body));
+  }
+
+  return bodies;
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function restCallWrapper(
+  request: () => Promise<any>,
+  retries: number = 3,
+  name: string = "-"
+) {
+  while (retries > 0) {
+    try {
+      const res = await request();
+      return res;
+    } catch {
+      await sleep(5_000 + 10_000 * Math.random());
+      restCallWrapper(request, retries--, name);
+    }
+  }
+  throw new Error(`couldnt work ${name} call after retries!`);
+}
+
+const priceCache: { [PK: string]: any } = {
+  "coingecko:tether": {
+    price: 1,
+    symbol: "USDT",
+    timestamp: Math.floor(Date.now() / 1e3 + 3600), // an hour from script start time
+  },
+};
+
+export async function getPrices(
+  readKeys: string[],
+  timestamp: number | "now"
+): Promise<{ [address: string]: CoinsApiData }> {
+  if (!readKeys.length) return {};
+
+  const aggregatedRes: { [address: string]: CoinsApiData } = {};
+
+  // read data from cache where possible
+  readKeys = readKeys.filter((PK: string) => {
+    if (timestamp !== "now") return true;
+    if (priceCache[PK]) {
+      aggregatedRes[PK] = { ...priceCache[PK], PK };
+      return false;
+    }
+    return true;
+  });
+
+  const bodies = getBodies(readKeys, timestamp);
+  const tokenData: CoinsApiData[][] = [];
+  await runInPromisePool({
+    items: bodies,
+    concurrency: 10,
+    processor: async (body: string) => {
+      const res = await restCallWrapper(() =>
+        axios.post(
+          `https://coins.llama.fi/prices?source=internal${coinsApiKey ? `?apikey=${coinsApiKey}` : ""
+          }`,
+          body,
+          {
+            headers: { "Content-Type": "application/json" },
+            params: { source: "internal", apikey: coinsApiKey },
+          },
+        )
+      );
+
+      const data = (res.data.coins = Object.entries(res.data.coins).map(
+        ([PK, value]) => ({
+          ...(value as CoinsApiData),
+          PK,
+        })
+      ));
+
+      tokenData.push(data);
+    },
+  });
+
+  const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase());
+  tokenData.map((batch: CoinsApiData[]) => {
+    batch.map((a: CoinsApiData) => {
+      if (!a.PK) return;
+      const i = normalizedReadKeys.indexOf(a.PK.toLowerCase());
+      aggregatedRes[readKeys[i]] = a;
+    });
+  });
+
+  return aggregatedRes;
+}
+
+const mcapCache: { [PK: string]: any } = {};
+
+export async function getMcaps(
+  readKeys: string[],
+  timestamp: number | "now"
+): Promise<{ [address: string]: McapsApiData }> {
+  if (!readKeys.length) return {};
+
+  const aggregatedRes: { [address: string]: McapsApiData } = {};
+
+  // read data from cache where possible
+  readKeys = readKeys.filter((PK: string) => {
+    if (timestamp !== "now") return true;
+    if (mcapCache[PK]) {
+      aggregatedRes[PK] = { ...mcapCache[PK], PK };
+      return false;
+    }
+    return true;
+  });
+
+  const bodies = getBodies(readKeys, timestamp);
+  const tokenData: { [key: string]: McapsApiData }[] = [];
+  await runInPromisePool({
+    items: bodies,
+    concurrency: 10,
+    processor: async (body: string) => {
+      const res = await restCallWrapper(() =>
+        axios.post(
+          `https://coins.llama.fi/mcaps${coinsApiKey ? `?apikey=${coinsApiKey}` : ""
+          }`,
+          body,
+          {
+            headers: { "Content-Type": "application/json" },
+          }
+        )
+      );
+      tokenData.push(res.data as any);
+    },
+  });
+
+  const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase());
+  tokenData.map((batch: { [key: string]: McapsApiData }) => {
+    Object.keys(batch).map((a: string) => {
+      if (!batch[a].mcap) return;
+      const i = normalizedReadKeys.indexOf(a.toLowerCase());
+      aggregatedRes[readKeys[i]] = batch[a];
+    });
+  });
+
+  return aggregatedRes;
+}

src/util/computeTVL.ts

@@ -1,7 +1,6 @@
-import { sliceIntoChunks } from ".";
-import { fetchJson, postJson, sumSingleBalance } from "./common";
+import { sumSingleBalance } from "../generalUtil";
 import { Balances } from "../types";
-import { ENV_CONSTANTS } from "./env";
+import { getPrices } from "./coinsApi";
 
 type PricesObject = {
   // NOTE: the tokens queried might be case sensitive and can be in mixed case, but while storing them in the cache, we convert them to lowercase
@@ -84,27 +83,11 @@ async function updatePriceCache(keys: string[], timestamp?: number) {
 
   const missingKeys = keys.filter(key => !pricesCache[key.toLowerCase()])
 
-  const chunks = sliceIntoChunks(missingKeys, 100)
-  for (const chunk of chunks) {
-    const coins = await getPrices(chunk)
-    for (const [token, data] of Object.entries(coins)) {
-      pricesCache[token.toLowerCase()] = data
-    }
-    chunk.map(i => i.toLowerCase()).filter(i => !pricesCache[i]).forEach(i => pricesCache[i] = {})
+  const coins = await getPrices(missingKeys, timestamp ?? "now")
+  for (const [token, data] of Object.entries(coins)) {
+    pricesCache[token.toLowerCase()] = data
   }
-
-  async function getPrices(keys: string[]) {
-    if (!timestamp) {
-      const { coins } = await fetchJson(`https://coins.llama.fi/prices/current/${keys.join(',')}`)
-      return coins
-    }
-
-    // fetch post with timestamp in body
-    const coinsApiKey = ENV_CONSTANTS['COINS_API_KEY']
-    const { coins } = await postJson("https://coins.llama.fi/prices?source=internal&apikey=" + coinsApiKey, { coins: keys, timestamp })
-    return coins
-  }
-
+  missingKeys.map(i => i.toLowerCase()).filter(i => !pricesCache[i]).forEach(i => pricesCache[i] = {})
 }
 
 function getPriceCache(timestamp?: number) {