Skip to content

Commit f41fac3

Browse files
waynebruce0xg1nt0ki
waynebruce0x
and
g1nt0ki
authored
Coins Api (#187)
* coins-api in SDK * rest call wrapper and key * cached queries SDK * Revert "cached queries SDK" This reverts commit 28bfd38. * cached * test * rename coins module --------- Co-authored-by: g1nt0ki <[email protected]>
1 parent 20bf57e commit f41fac3

File tree

6 files changed

+202
-24
lines changed

6 files changed

+202
-24
lines changed

‎src/api2.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ export * as util from "./util";
22
export * as eth from "./eth";
33
export * as erc20 from "./erc20";
44
export * as abi from "./abi/abi2";
5-
export * as config from "./general";
5+
export * as config from "./general";

‎src/index.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ test("imports", async () => {
116116
"writeCache": [Function],
117117
"writeExpiringJsonCache": [Function],
118118
},
119+
"coins": {
120+
"getMcaps": [Function],
121+
"getPrices": [Function],
122+
},
119123
"elastic": {
120124
"addDebugLog": [Function],
121125
"addErrorLog": [Function],

‎src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export * as indexer from "./util/indexer";
1717
export * as types from "./types";
1818
export * as tron from "./abi/tron";
1919
export * as erc20 from "./erc20";
20+
export * as coins from "./util/coinsApi";
2021

2122
export const log = debugLog
2223
export const logTable = debugTable

‎src/util/coinsApi.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { getPrices, getMcaps } from "./coinsApi";
2+
3+
test("coinsApi - mcaps", async () => {
4+
const res = await getMcaps(["coingecko:tether"], "now");
5+
expect(res["coingecko:tether"].mcap).toBeGreaterThan(100_000);
6+
expect(res["coingecko:tether"].mcap).toBeLessThan(1_000_000_000_000);
7+
expect(res["coingecko:tether"].timestamp).toBeGreaterThan(Math.floor(Date.now() / 1e3 - 3600));
8+
expect(Object.keys(res).length).toBe(1);
9+
})
10+
11+
test("coinsApi - prices", async () => {
12+
const res = await getPrices(["coingecko:tether", "ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7", "solana:So11111111111111111111111111111111111111112"], "now");
13+
expect(res["coingecko:tether"].price).toBe(1);
14+
expect(res["ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7"].symbol).toBe("USDT");
15+
expect(res["ethereum:0xdac17f958d2ee523a2206206994597c13d831ec7"].decimals).toBe(6);
16+
expect(Object.keys(res).length).toBe(3);
17+
expect(res["solana:So11111111111111111111111111111111111111112"].timestamp).toBeGreaterThan(Math.floor(Date.now() / 1e3 - 3600));
18+
})

‎src/util/coinsApi.ts‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
import axios from "axios";
2+
import { getEnvValue } from "./env";
3+
import runInPromisePool from "./promisePool";
4+
5+
type CoinsApiData = {
6+
decimals: number;
7+
price: number;
8+
symbol: string;
9+
timestamp: number;
10+
PK?: string;
11+
};
12+
13+
type McapsApiData = {
14+
mcap: number;
15+
timestamp: number;
16+
};
17+
18+
const coinsApiKey = getEnvValue("COINS_API_KEY")
19+
const bodySize = 2; // 100;
20+
21+
function getBodies(readKeys: string[], timestamp: number | "now") {
22+
const bodies: string[] = [];
23+
for (let i = 0; i < readKeys.length; i += bodySize) {
24+
const body = {
25+
coins: readKeys.slice(i, i + bodySize),
26+
} as any;
27+
if (timestamp !== "now") body.timestamp = timestamp;
28+
bodies.push(JSON.stringify(body));
29+
}
30+
31+
return bodies;
32+
}
33+
34+
function sleep(ms: number) {
35+
return new Promise((resolve) => setTimeout(resolve, ms));
36+
}
37+
38+
async function restCallWrapper(
39+
request: () => Promise<any>,
40+
retries: number = 3,
41+
name: string = "-"
42+
) {
43+
while (retries > 0) {
44+
try {
45+
const res = await request();
46+
return res;
47+
} catch {
48+
await sleep(5_000 + 10_000 * Math.random());
49+
restCallWrapper(request, retries--, name);
50+
}
51+
}
52+
throw new Error(`couldnt work ${name} call after retries!`);
53+
}
54+
55+
const priceCache: { [PK: string]: any } = {
56+
"coingecko:tether": {
57+
price: 1,
58+
symbol: "USDT",
59+
timestamp: Math.floor(Date.now() / 1e3 + 3600), // an hour from script start time
60+
},
61+
};
62+
63+
export async function getPrices(
64+
readKeys: string[],
65+
timestamp: number | "now"
66+
): Promise<{ [address: string]: CoinsApiData }> {
67+
if (!readKeys.length) return {};
68+
69+
const aggregatedRes: { [address: string]: CoinsApiData } = {};
70+
71+
// read data from cache where possible
72+
readKeys = readKeys.filter((PK: string) => {
73+
if (timestamp !== "now") return true;
74+
if (priceCache[PK]) {
75+
aggregatedRes[PK] = { ...priceCache[PK], PK };
76+
return false;
77+
}
78+
return true;
79+
});
80+
81+
const bodies = getBodies(readKeys, timestamp);
82+
const tokenData: CoinsApiData[][] = [];
83+
await runInPromisePool({
84+
items: bodies,
85+
concurrency: 10,
86+
processor: async (body: string) => {
87+
const res = await restCallWrapper(() =>
88+
axios.post(
89+
`https://coins.llama.fi/prices?source=internal${coinsApiKey ? `?apikey=${coinsApiKey}` : ""
90+
}`,
91+
body,
92+
{
93+
headers: { "Content-Type": "application/json" },
94+
params: { source: "internal", apikey: coinsApiKey },
95+
},
96+
)
97+
);
98+
99+
const data = (res.data.coins = Object.entries(res.data.coins).map(
100+
([PK, value]) => ({
101+
...(value as CoinsApiData),
102+
PK,
103+
})
104+
));
105+
106+
tokenData.push(data);
107+
},
108+
});
109+
110+
const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase());
111+
tokenData.map((batch: CoinsApiData[]) => {
112+
batch.map((a: CoinsApiData) => {
113+
if (!a.PK) return;
114+
const i = normalizedReadKeys.indexOf(a.PK.toLowerCase());
115+
aggregatedRes[readKeys[i]] = a;
116+
});
117+
});
118+
119+
return aggregatedRes;
120+
}
121+
122+
const mcapCache: { [PK: string]: any } = {};
123+
124+
export async function getMcaps(
125+
readKeys: string[],
126+
timestamp: number | "now"
127+
): Promise<{ [address: string]: McapsApiData }> {
128+
if (!readKeys.length) return {};
129+
130+
const aggregatedRes: { [address: string]: McapsApiData } = {};
131+
132+
// read data from cache where possible
133+
readKeys = readKeys.filter((PK: string) => {
134+
if (timestamp !== "now") return true;
135+
if (mcapCache[PK]) {
136+
aggregatedRes[PK] = { ...mcapCache[PK], PK };
137+
return false;
138+
}
139+
return true;
140+
});
141+
142+
const bodies = getBodies(readKeys, timestamp);
143+
const tokenData: { [key: string]: McapsApiData }[] = [];
144+
await runInPromisePool({
145+
items: bodies,
146+
concurrency: 10,
147+
processor: async (body: string) => {
148+
const res = await restCallWrapper(() =>
149+
axios.post(
150+
`https://coins.llama.fi/mcaps${coinsApiKey ? `?apikey=${coinsApiKey}` : ""
151+
}`,
152+
body,
153+
{
154+
headers: { "Content-Type": "application/json" },
155+
}
156+
)
157+
);
158+
tokenData.push(res.data as any);
159+
},
160+
});
161+
162+
const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase());
163+
tokenData.map((batch: { [key: string]: McapsApiData }) => {
164+
Object.keys(batch).map((a: string) => {
165+
if (!batch[a].mcap) return;
166+
const i = normalizedReadKeys.indexOf(a.toLowerCase());
167+
aggregatedRes[readKeys[i]] = batch[a];
168+
});
169+
});
170+
171+
return aggregatedRes;
172+
}

‎src/util/computeTVL.ts‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
import { sliceIntoChunks } from ".";
2-
import { fetchJson, postJson, sumSingleBalance } from "./common";
1+
import { sumSingleBalance } from "../generalUtil";
32
import { Balances } from "../types";
4-
import { ENV_CONSTANTS } from "./env";
3+
import { getPrices } from "./coinsApi";
54

65
type PricesObject = {
76
// NOTE: the tokens queried might be case sensitive and can be in mixed case, but while storing them in the cache, we convert them to lowercase
@@ -84,27 +83,11 @@ async function updatePriceCache(keys: string[], timestamp?: number) {
8483

8584
const missingKeys = keys.filter(key => !pricesCache[key.toLowerCase()])
8685

87-
const chunks = sliceIntoChunks(missingKeys, 100)
88-
for (const chunk of chunks) {
89-
const coins = await getPrices(chunk)
90-
for (const [token, data] of Object.entries(coins)) {
91-
pricesCache[token.toLowerCase()] = data
92-
}
93-
chunk.map(i => i.toLowerCase()).filter(i => !pricesCache[i]).forEach(i => pricesCache[i] = {})
86+
const coins = await getPrices(missingKeys, timestamp ?? "now")
87+
for (const [token, data] of Object.entries(coins)) {
88+
pricesCache[token.toLowerCase()] = data
9489
}
95-
96-
async function getPrices(keys: string[]) {
97-
if (!timestamp) {
98-
const { coins } = await fetchJson(`https://coins.llama.fi/prices/current/${keys.join(',')}`)
99-
return coins
100-
}
101-
102-
// fetch post with timestamp in body
103-
const coinsApiKey = ENV_CONSTANTS['COINS_API_KEY']
104-
const { coins } = await postJson("https://coins.llama.fi/prices?source=internal&apikey=" + coinsApiKey, { coins: keys, timestamp })
105-
return coins
106-
}
107-
90+
missingKeys.map(i => i.toLowerCase()).filter(i => !pricesCache[i]).forEach(i => pricesCache[i] = {})
10891
}
10992

11093
function getPriceCache(timestamp?: number) {

0 commit comments

Comments
 (0)