wok on kbin

crawler/src/crawl/fediseer.ts

@@ -1,6 +1,7 @@
 import logging from "../lib/logging";
 import storage from "../lib/crawlStorage";
-import { IFediseerInstanceData, IFediseerTag } from "../lib/storage/fediseer";
+
+import { IFediseerInstanceData, IFediseerTag } from "../../../types/storage";
 
 import CrawlClient from "../lib/CrawlClient";
 

crawler/src/crawl/instance.ts

@@ -5,7 +5,7 @@ import {
   IErrorDataKeyValue,
   ILastCrawlData,
   ILastCrawlDataKeyValue,
-} from "../lib/storage/tracking";
+} from "../../../types/storage";
 
 import { CRAWL_AGED_TIME } from "../lib/const";
 import { HTTPError, CrawlError, CrawlTooRecentError } from "../lib/error";

crawler/src/crawl/mbin.ts

@@ -5,8 +5,8 @@ import { exec } from "node:child_process";
 import logging from "../lib/logging";
 
 import storage from "../lib/crawlStorage";
-import { IFediverseDataKeyValue } from "../lib/storage/fediverse";
-import { IMagazineData } from "../lib/storage/mbin";
+import { IFediverseDataKeyValue } from "../../../types/storage";
+import { IMagazineData } from "../../../types/storage";
 
 import { CrawlError, CrawlTooRecentError } from "../lib/error";
 
@@ -311,6 +311,8 @@ export default class CrawlMBin {
     const outMagazineData: IMagazineData = {
       baseurl: crawlDomain,
       ...magazineData,
+
+      icon: magazineData.icon.storageUrl ? magazineData.icon.storageUrl : null,
       lastCrawled: Date.now(),
     };
 

crawler/src/lib/storage/community.ts

@@ -1,34 +1,6 @@
 import { CrawlStorage } from "../crawlStorage";
 
-export type ICommunityData = {
-  community: {
-    id: number;
-    name: string;
-    title: string;
-    description: string;
-    removed: boolean;
-    published: string;
-    updated: string | null;
-    deleted: boolean;
-    nsfw: boolean;
-    actor_id: string;
-    local: boolean;
-    icon: string | null;
-    banner: string | null;
-    hidden: boolean;
-    posting_restricted_to_mods: boolean;
-    instance_id: number;
-  };
-  subscribed: string;
-  blocked: boolean;
-  counts: Object;
-  banned_from_community?: boolean;
-  lastCrawled: number;
-};
-
-export type ICommunityDataKeyValue = {
-  [key: string]: ICommunityData;
-};
+import { ICommunityData, ICommunityDataKeyValue } from "../../../../types/storage";
 
 export default class Community {
   private storage: CrawlStorage;

crawler/src/lib/storage/fediseer.ts

@@ -1,43 +1,6 @@
 import { CrawlStorage } from "../crawlStorage";
 
-export type IFediseerInstanceFlags = {
-  flag: "RESTRICTED" | "MUTED";
-  comment: string;
-};
-
-export type IFediseerTag = {
-  tag: string;
-  count?: number;
-  rank?: number;
-};
-
-export type IFediseerInstanceData = {
-  id: number;
-  domain: string;
-  software: string;
-  version: string;
-  claimed: number;
-  open_registrations: boolean;
-  email_verify: boolean;
-  approval_required: boolean;
-  has_captcha: boolean;
-  approvals: number;
-  endorsements: number;
-  guarantor: string;
-  censure_reasons: string[] | null;
-  sysadmins: number;
-  moderators: number;
-
-  state: "UP" | "UNREACHABLE" | "OFFLINE" | "DECOMMISSIONED";
-
-  tags: IFediseerTag[] | string[];
-
-  visibility_endorsements: "OPEN" | "ENDORSED" | "PRIVATE";
-  visibility_censures: "OPEN" | "ENDORSED" | "PRIVATE";
-  visibility_hesitations: "OPEN" | "ENDORSED" | "PRIVATE";
-
-  flags: IFediseerInstanceFlags[];
-};
+import { IFediseerInstanceData } from "../../../../types/storage";
 
 export default class Fediseer {
   private storage: CrawlStorage;

crawler/src/lib/storage/fediverse.ts

@@ -1,17 +1,6 @@
 import { CrawlStorage } from "../crawlStorage";
 
-export type IFediverseData = {
-  time?: number;
-  baseurl?: string;
-  name?: string;
-  version?: string;
-  repository?: string;
-  homepage?: string;
-};
-
-export type IFediverseDataKeyValue = {
-  [key: string]: IFediverseData;
-};
+import { IFediverseData, IFediverseDataKeyValue } from "../../../../types/storage";
 
 export default class Fediverse {
   private storage: CrawlStorage;

crawler/src/lib/storage/instance.ts

@@ -1,23 +1,13 @@
 import { CrawlStorage } from "../crawlStorage";
 
+import { IInstanceData, IInstanceDataKeyValue } from "../../../../types/storage";
+
 /**
  * Stores each lemmy instance, keyed on baseUrl as `instance:baseUrl`.
  *
  * Each instance is stored as a JSON object with the following fields:
  */
 
-export type IInstanceData = {
-  nodeData: any;
-  siteData: any;
-  headers: any;
-  langs: Array<string>;
-  lastCrawled: number;
-};
-
-export type IInstanceDataKeyValue = {
-  [key: string]: IInstanceData;
-};
-
 export default class Instance {
   private storage: CrawlStorage;
 

crawler/src/lib/storage/mbin.ts

@@ -1,57 +1,6 @@
 import { CrawlStorage } from "../crawlStorage";
 
-export type IMagazineData = {
-  magazineId: number;
-  owner: {
-    magazineId: number;
-    userId: number;
-    avatar: any;
-    username: string;
-    apId: any;
-  };
-  icon: {
-    storageUrl: string;
-    [key: string]: any;
-  };
-  name: string;
-  title: string;
-  description: string;
-  rules: string;
-  subscriptionsCount: number;
-  entryCount: number;
-  entryCommentCount: number;
-  postCount: number;
-  postCommentCount: number;
-  isAdult: boolean;
-  isUserSubscribed: any;
-  isBlockedByUser: any;
-  tags: string[];
-  badges: {
-    badgeId: number;
-    magazineId: number;
-    name: string;
-  }[];
-  moderators: {
-    magazineId: number;
-    userId: number;
-    avatar: {
-      storageUrl: string;
-      [key: string]: any;
-    };
-    username: string;
-    apId: any;
-  }[];
-  apId: any;
-  apProfileId: string;
-  serverSoftware: any;
-  serverSoftwareVersion: any;
-  isPostingRestrictedToMods: boolean;
-  lastCrawled?: number;
-  baseurl: string;
-};
-export type IMagazineDataKeyValue = {
-  [key: string]: IMagazineData;
-};
+import { IMagazineData, IMagazineDataKeyValue } from "../../../../types/storage";
 
 export default class MBinStore {
   private storage: CrawlStorage;

crawler/src/lib/storage/tracking.ts

@@ -2,30 +2,12 @@ import { CrawlStorage } from "../crawlStorage";
 
 import { RECORD_TTL_TIMES_SECONDS } from "../const";
 
-export type IErrorData = {
-  time: number;
-  error: string;
-  stack?: string;
-  isAxiosError?: boolean;
-  requestUrl?: string;
-  code?: string;
-  url?: string;
-  duration?: number;
-};
-
-export type IErrorDataKeyValue = {
-  [key: string]: IErrorData;
-};
-
-export type ILastCrawlData = {
-  time: number;
-  duration?: number;
-  [key: string]: any;
-};
-
-export type ILastCrawlDataKeyValue = {
-  [key: string]: ILastCrawlData;
-};
+import {
+  IErrorData,
+  IErrorDataKeyValue,
+  ILastCrawlData,
+  ILastCrawlDataKeyValue,
+} from "../../../../types/storage";
 
 export default class TrackingStore {
   private storage: CrawlStorage;

crawler/src/lib/storage/uptime.ts

@@ -1,21 +1,6 @@
 import { CrawlStorage } from "../crawlStorage";
 
-export type IUptimeNodeData = {
-  domain: string;
-  latency: number;
-  countryname: string;
-  uptime_alltime: string;
-  date_created: string;
-  date_updated: string;
-  date_laststats: string;
-  score: number;
-  status: number;
-};
-
-export type IFullUptimeData = {
-  timestamp: number;
-  nodes: IUptimeNodeData[];
-};
+import { IUptimeNodeData, IFullUptimeData } from "../../../../types/storage";
 
 export default class Uptime {
   private storage: CrawlStorage;