Aggregate past data even for removed pools

README.md

@@ -45,6 +45,25 @@ interface TokenAmount {
 }
 ```
 
+# Include data of a removed pool
+
+Only supported on Solana.
+By default removed pools are not added to aggregated TVL, Volume, Fee intervals.
+To include them add an entry in `data/removed_pools_mainnet.json` following the format:
+Adding pools that not have been removed to the file has no adverse effect on them, it exists to supply information not available on-chain after removal.
+
+```
+{
+  "poolAddress": {
+    "tokenX": string,
+    "tokenY": string,
+    "tickSpacing": number,
+    "fee": string
+  }
+}
+
+```
+
 # Generated data
 
 List of snapshot files:

data/removed_pools_mainnet.json

@@ -0,0 +1 @@
+{}

solana/src/aggregate-intervals.ts

@@ -12,6 +12,7 @@ import ARCHIVAL_DATA from "../../data/archive/solana_mainnet.json";
 import MAINNET_DATA from "../../data/mainnet.json";
 import DEVNET_APY_ARCHIVE from "../../data/daily_pool_apy_devnet.json";
 import MAINNET_APY_ARCHIVE from "../../data/daily_pool_apy_mainnet.json";
+import MAINNET_REMOVED_POOLS from "../../data/removed_pools_mainnet.json";
 import {
   isSameWeek,
   PoolStatsData,
@@ -34,9 +35,10 @@ import {
   getIntervalRange,
   getTokensPriceFeed,
   getEmptyIntervalsData,
+  PoolSignature,
 } from "./utils";
 import { DECIMAL } from "@invariant-labs/sdk/lib/utils";
-import { Provider } from "@project-serum/anchor";
+import { BN, Provider } from "@project-serum/anchor";
 import { PoolStructure } from "@invariant-labs/sdk/lib/market";
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 require("dotenv").config();
@@ -63,6 +65,7 @@ export const createSnapshotForNetwork = async (network: Network) => {
   let snaps: Record<string, PoolStatsData>;
   let archivalSnaps: Record<string, PoolStatsData>;
   let apy: Record<string, number>;
+  let removedPools: Record<string, PoolSignature>;
   let poolsCacheFileName: string;
 
   const args = process.argv.slice(2);
@@ -75,6 +78,7 @@ export const createSnapshotForNetwork = async (network: Network) => {
       snaps = DEVNET_DATA;
       archivalSnaps = {};
       apy = DEVNET_APY_ARCHIVE;
+      removedPools = {};
       poolsCacheFileName = "../data/cache/devnet_pools_cache.json";
       intervalsPath = "../data/intervals/devnet/";
       fileName = "../data/devnet_intervals.json";
@@ -92,6 +96,7 @@ export const createSnapshotForNetwork = async (network: Network) => {
       // @ts-ignore
       archivalSnaps = ARCHIVAL_DATA;
       apy = MAINNET_APY_ARCHIVE;
+      removedPools = MAINNET_REMOVED_POOLS;
       intervalsPath = "../data/intervals/mainnet/";
       fileName = "../data/mainnet_intervals.json";
       break;
@@ -184,7 +189,14 @@ export const createSnapshotForNetwork = async (network: Network) => {
     //   continue;
     // }
 
-    const pool = poolsMapping.get(poolKey);
+    const pool:
+      | {
+          tokenX: PublicKey;
+          tokenY: PublicKey;
+          tickSpacing: number;
+          fee: { v: BN };
+        }
+      | undefined = getPoolKeySignature(poolsMapping, removedPools, poolKey);
 
     if (!pool) {
       continue;
@@ -586,6 +598,27 @@ export const createSnapshotForNetwork = async (network: Network) => {
   fs.writeFileSync(fileName, JSON.stringify(totalStats), "utf-8");
 };
 
+const getPoolKeySignature = (
+  poolsMapping: Map<string, PoolStructure>,
+  removedPools: Record<string, PoolSignature>,
+  poolKey: string
+) => {
+  const pool = poolsMapping.get(poolKey);
+  if (pool) {
+    return pool;
+  }
+
+  if (removedPools[poolKey]) {
+    const poolSig = removedPools[poolKey];
+    return {
+      tokenX: new PublicKey(poolSig.tokenX),
+      tokenY: new PublicKey(poolSig.tokenY),
+      tickSpacing: poolSig.tickSpacing,
+      fee: { v: new BN(poolSig.fee) },
+    };
+  }
+};
+
 // createSnapshotForNetwork(Network.DEV).then(
 //   () => {
 //     console.log('Eclipse: Devnet pool apy snapshot done!')

solana/src/utils.ts

@@ -5,7 +5,7 @@ import { Market, PoolStructure, Tick } from "@invariant-labs/sdk/lib/market";
 import { Market as EclipseMarket } from "@invariant-labs/sdk-eclipse/lib/market";
 import { DECIMAL, Range } from "@invariant-labs/sdk/lib/utils";
 import BN from "bn.js";
-import { Connection, ParsedAccountData, PublicKey } from "@solana/web3.js";
+import { Connection, PublicKey } from "@solana/web3.js";
 import axios, { AxiosResponse } from "axios";
 // @ts-ignore
 import MAINNET_TOKENS from "../../data/mainnet_tokens.json";
@@ -107,6 +107,16 @@ export interface PoolStatsData {
   };
 }
 
+export interface PoolSignature {
+  tokenX: string;
+
+  tokenY: string;
+
+  tickSpacing: number;
+
+  fee: string;
+}
+
 export interface PoolsApyStatsData {
   apy: number;
   weeklyFacetor: number[];