output script changes

tgxn · Jan 7, 2025 · cda3632 · cda3632
1 parent 20aaa79
commit cda3632
Show file tree

Hide file tree

Showing 7 changed files with 304 additions and 290 deletions.
diff --git a/crawler/src/lib/const.ts b/crawler/src/lib/const.ts
@@ -38,6 +38,8 @@ export const CRAWL_AGED_TIME = {
 
   // if a server is identified as a non-lemmy server, ho often should we wait before checking again?
   FEDIVERSE: hoursToMs(2 * 24), // 2 days
+
+  MAGAZINE: hoursToMs(8),
 };
 
 // consider for deletion after they haven't been seen for this long

diff --git a/crawler/src/lib/crawlStorage.ts b/crawler/src/lib/crawlStorage.ts
@@ -6,7 +6,6 @@ import logging from "./logging";
 // core
 import InstanceStore from "./storage/instance";
 import CommunityStore from "./storage/community";
-import KBinStore from "./storage/kbin";
 import MBinStore from "./storage/mbin";
 
 // supporting
@@ -29,7 +28,6 @@ export class CrawlStorage {
   public fediverse: FediverseStore;
   public fediseer: FediseerStore;
   public tracking: TrackingStore;
-  public kbin: KBinStore;
   public mbin: MBinStore;
 
   constructor() {
@@ -46,7 +44,6 @@ export class CrawlStorage {
     this.fediverse = new FediverseStore(this);
     this.fediseer = new FediseerStore(this);
     this.tracking = new TrackingStore(this);
-    this.kbin = new KBinStore(this);
     this.mbin = new MBinStore(this);
   }
 

diff --git a/crawler/src/lib/storage/kbin.ts b/crawler/src/lib/storage/kbin.ts
diff --git a/crawler/src/lib/storage/mbin.ts b/crawler/src/lib/storage/mbin.ts
@@ -37,6 +37,7 @@ export type IMagazineData = {
   serverSoftwareVersion: any;
   isPostingRestrictedToMods: boolean;
   lastCrawled?: number;
+  baseurl: string;
 };
 export type IMagazineDataKeyValue = {
   [key: string]: IMagazineData;
@@ -50,7 +51,14 @@ export default class MBinStore {
   }
 
   async getAll(): Promise<IMagazineData[]> {
-    return this.storage.listRedis(`mbin_magazine:*`);
+    const magazineKeyValue = this.storage.listRedisWithKeys(`mbin_magazine:*`);
+
+    // put baseUrl into the magazine object
+    for (const key in magazineKeyValue) {
+      magazineKeyValue[key].baseurl = key.split(":")[1];
+    }
+
+    return Object.values(magazineKeyValue);
   }
 
   async getAllWithKeys(): Promise<IMagazineDataKeyValue> {