add fediseer types

tgxn · Jan 3, 2025 · 80ac6e2 · 80ac6e2
1 parent 32ceaad
commit 80ac6e2
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 26 deletions.
diff --git a/crawler/src/crawl/fediseer.ts b/crawler/src/crawl/fediseer.ts
@@ -1,5 +1,6 @@
 import logging from "../lib/logging";
 import storage from "../lib/crawlStorage";
+import { IFediseerInstanceData, IFediseerTag } from "../lib/storage/fediseer";
 
 import CrawlClient from "../lib/CrawlClient";
 
@@ -10,30 +11,14 @@ export default class CrawlFediseer {
     this.client = new CrawlClient("https://fediseer.com/api/v1/");
   }
 
-  async getAllPagesData(page = 1) {
-    let mergedInstances: any = [];
+  async getAllPagesData(page: number = 1): Promise<IFediseerInstanceData[]> {
+    let mergedInstances: IFediseerInstanceData[] = [];
 
-    const perPage = 100;
+    const perPage: number = 100;
 
     const fediseerWhitelist = await this.client.getUrl(
       `/whitelist?endorsements=0&guarantors=0&page=${page}&limit=${perPage}&software_csv=lemmy`,
     );
-    //   //  {
-    //   //   query: `query{
-    //   //         nodes (softwarename: "lemmy") {
-    //   //         domain
-    //   //         latency
-    //   //         countryname
-    //   //         uptime_alltime
-    //   //         date_created
-    //   //         date_updated
-    //   //         date_laststats
-    //   //         score
-    //   //         status
-    //   //         }
-    //   //     }`,
-    //   // }
-    // );
 
     mergedInstances = [...fediseerWhitelist.data.instances];
 
@@ -76,7 +61,9 @@ export default class CrawlFediseer {
     // map "rank" into tags, based on data from fediseerTopTagsData [{ tag, count }]
     fediseerWhitelist.forEach((instance) => {
       instance.tags = instance.tags.map((tag) => {
-        const fediseerTag = fediseerTopTagsData.data.find((fediseerTag) => fediseerTag.tag == tag);
+        const fediseerTag = fediseerTopTagsData.data.find(
+          (fediseerTag: IFediseerTag) => fediseerTag.tag == tag,
+        );
 
         if (!fediseerTag) console.log("fediseerTag not found in top tags", tag, fediseerTag);
 

diff --git a/crawler/src/lib/storage/fediseer.ts b/crawler/src/lib/storage/fediseer.ts
@@ -30,22 +30,61 @@ import { CrawlStorage } from "../crawlStorage";
     },
  */
 
-export type IFediseerTagData = {
+// export type IFediseerTagData = {
+//   tag: string;
+//   rank: number;
+// };
+
+// export type IFediseerData = {
+//   id: number;
+//   domain: string;
+//   software: string;
+//   claimed: number;
+//   open_registrations: boolean;
+//   email_verify: boolean;
+//   approvals: number;
+//   endorsements: number;
+//   guarantor: string;
+//   tags?: IFediseerTagData[];
+// };
+
+export type IFediseerInstanceFlags = {
+  flag: "RESTRICTED" | "MUTED";
+  comment: string;
+};
+
+export type IFediseerTag = {
   tag: string;
-  rank: number;
+  count?: number;
+  rank?: number;
 };
 
-export type IFediseerData = {
+export type IFediseerInstanceData = {
   id: number;
   domain: string;
   software: string;
+  version: string;
   claimed: number;
   open_registrations: boolean;
   email_verify: boolean;
+  approval_required: boolean;
+  has_captcha: boolean;
   approvals: number;
   endorsements: number;
   guarantor: string;
-  tags?: IFediseerTagData[];
+  censure_reasons: string[] | null;
+  sysadmins: number;
+  moderators: number;
+
+  state: "UP" | "UNREACHABLE" | "OFFLINE" | "DECOMMISSIONED";
+
+  tags: IFediseerTag[] | string[];
+
+  visibility_endorsements: "OPEN" | "ENDORSED" | "PRIVATE";
+  visibility_censures: "OPEN" | "ENDORSED" | "PRIVATE";
+  visibility_hesitations: "OPEN" | "ENDORSED" | "PRIVATE";
+
+  flags: IFediseerInstanceFlags[];
 };
 
 export default class Fediseer {
@@ -55,14 +94,14 @@ export default class Fediseer {
     this.storage = storage;
   }
 
-  async getLatest(): Promise<IFediseerData[]> {
+  async getLatest(): Promise<IFediseerInstanceData[]> {
     // records have uptime:timestamp key, extract the latest one
     const keys = await this.storage.client.keys(`fediseer:*`);
     const latestKey = keys.reduce((a, b) => (a > b ? a : b));
     return this.storage.getRedis(latestKey);
   }
 
-  async addNew(data: IFediseerData[]) {
+  async addNew(data: IFediseerInstanceData[]) {
     return this.storage.putRedis(`fediseer:${Date.now()}`, data);
   }
 }