twitter db

Dockerfile

@@ -51,6 +51,7 @@ COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs
 RUN mkdir -p /litefs /var/lib/litefs /public && \
     chown -R bun:bun /litefs /var/lib/litefs /public
 
+# Create volume mount points
 # Copy only necessary files from builders
 COPY --from=backend-builder --chown=bun:bun /app/package.json ./
 COPY --chown=bun:bun curate.config.json ./
@@ -60,7 +61,6 @@ COPY --from=backend-builder --chown=bun:bun /app/backend/dist ./backend/dist
 
 # Set environment variables
 ENV DATABASE_URL="file:/litefs/db"
-ENV CACHE_DIR="/litefs/cache"
 ENV NODE_ENV="production"
 
 # Expose the port

backend/src/services/db/index.ts

@@ -1,11 +1,15 @@
 import { Database } from "bun:sqlite";
 import { BunSQLiteDatabase, drizzle } from "drizzle-orm/bun-sqlite";
 import { join } from "node:path";
-import { Moderation, TwitterSubmission } from "types/twitter";
+
+import { logger } from "utils/logger";
 import { broadcastUpdate } from "../../index";
+
 import * as queries from "./queries";
-import { logger } from "utils/logger";
 
+// Twitter
+import { Moderation, TwitterSubmission, TwitterCookie } from "types/twitter";
+import * as twitterQueries from "../twitter/queries";
 export class DatabaseService {
   private db: BunSQLiteDatabase;
   private static readonly DB_PATH =
@@ -121,6 +125,46 @@ export class DatabaseService {
     // For now, content is the same as submission since we're dealing with tweets
     return this.getSubmission(contentId);
   }
+
+  // Twitter Cookie Management
+  setTwitterCookies(username: string, cookies: TwitterCookie[]): void {
+    const cookiesJson = JSON.stringify(cookies);
+    twitterQueries.setTwitterCookies(this.db, username, cookiesJson).run();
+  }
+
+  getTwitterCookies(username: string): TwitterCookie[] | null {
+    const result = twitterQueries.getTwitterCookies(this.db, username);
+    if (!result) return null;
+
+    try {
+      return JSON.parse(result.cookies) as TwitterCookie[];
+    } catch (e) {
+      logger.error("Error parsing Twitter cookies:", e);
+      return null;
+    }
+  }
+
+  deleteTwitterCookies(username: string): void {
+    twitterQueries.deleteTwitterCookies(this.db, username).run();
+  }
+
+  // Twitter Cache Management
+  setTwitterCacheValue(key: string, value: string): void {
+    twitterQueries.setTwitterCacheValue(this.db, key, value).run();
+  }
+
+  getTwitterCacheValue(key: string): string | null {
+    const result = twitterQueries.getTwitterCacheValue(this.db, key);
+    return result?.value ?? null;
+  }
+
+  deleteTwitterCacheValue(key: string): void {
+    twitterQueries.deleteTwitterCacheValue(this.db, key).run();
+  }
+
+  clearTwitterCache(): void {
+    twitterQueries.clearTwitterCache(this.db).run();
+  }
 }
 
 // Export a singleton instance

backend/src/services/db/queries.ts

@@ -1,13 +1,13 @@
 import { and, eq, sql } from "drizzle-orm";
 import { BunSQLiteDatabase } from "drizzle-orm/bun-sqlite";
+import { Moderation, TwitterSubmission } from "types/twitter";
 import {
+  feeds,
   moderationHistory,
   submissionCounts,
-  submissions,
-  feeds,
   submissionFeeds,
+  submissions
 } from "./schema";
-import { Moderation, TwitterSubmission } from "types/twitter";
 
 export function upsertFeed(
   db: BunSQLiteDatabase,
@@ -172,11 +172,11 @@ export function getSubmission(
     submittedAt: result.submittedAt,
     moderationHistory: result.moderationHistory
       ? JSON.parse(`[${result.moderationHistory}]`)
-          .filter((m: any) => m !== null)
-          .map((m: any) => ({
-            ...m,
-            timestamp: new Date(m.timestamp),
-          }))
+        .filter((m: any) => m !== null)
+        .map((m: any) => ({
+          ...m,
+          timestamp: new Date(m.timestamp),
+        }))
       : [],
   };
 }
@@ -223,11 +223,11 @@ export function getSubmissionByAcknowledgmentTweetId(
     submittedAt: result.submittedAt,
     moderationHistory: result.moderationHistory
       ? JSON.parse(`[${result.moderationHistory}]`)
-          .filter((m: any) => m !== null)
-          .map((m: any) => ({
-            ...m,
-            timestamp: new Date(m.timestamp),
-          }))
+        .filter((m: any) => m !== null)
+        .map((m: any) => ({
+          ...m,
+          timestamp: new Date(m.timestamp),
+        }))
       : [],
   };
 }
@@ -268,11 +268,11 @@ export function getAllSubmissions(db: BunSQLiteDatabase): TwitterSubmission[] {
     submittedAt: result.submittedAt,
     moderationHistory: result.moderationHistory
       ? JSON.parse(result.moderationHistory)
-          .filter((m: any) => m !== null)
-          .map((m: any) => ({
-            ...m,
-            timestamp: new Date(m.timestamp),
-          }))
+        .filter((m: any) => m !== null)
+        .map((m: any) => ({
+          ...m,
+          timestamp: new Date(m.timestamp),
+        }))
       : [],
   }));
 }
@@ -317,11 +317,11 @@ export function getSubmissionsByStatus(
     submittedAt: result.submittedAt,
     moderationHistory: result.moderationHistory
       ? JSON.parse(result.moderationHistory)
-          .filter((m: any) => m !== null)
-          .map((m: any) => ({
-            ...m,
-            timestamp: new Date(m.timestamp),
-          }))
+        .filter((m: any) => m !== null)
+        .map((m: any) => ({
+          ...m,
+          timestamp: new Date(m.timestamp),
+        }))
       : [],
   }));
 }
@@ -452,11 +452,11 @@ export function getSubmissionsByFeed(
     submittedAt: result.submittedAt,
     moderationHistory: result.moderationHistory
       ? JSON.parse(result.moderationHistory)
-          .filter((m: any) => m !== null)
-          .map((m: any) => ({
-            ...m,
-            timestamp: new Date(m.timestamp),
-          }))
+        .filter((m: any) => m !== null)
+        .map((m: any) => ({
+          ...m,
+          timestamp: new Date(m.timestamp),
+        }))
       : [],
   }));
 }

backend/src/services/db/schema.ts

@@ -6,6 +6,9 @@ import {
   text,
 } from "drizzle-orm/sqlite-core";
 
+// From exports/plugins
+export * from "../twitter/schema";
+
 // Reusable timestamp columns
 const timestamps = {
   createdAt: text("created_at")

backend/src/services/twitter/client.ts

@@ -1,13 +1,7 @@
 import { Scraper, SearchMode, Tweet } from "agent-twitter-client";
+import { TwitterCookie } from "types/twitter";
 import { logger } from "../../utils/logger";
-import {
-  TwitterCookie,
-  cacheCookies,
-  ensureCacheDirectory,
-  getCachedCookies,
-  getLastCheckedTweetId,
-  saveLastCheckedTweetId,
-} from "../../utils/cache";
+import { db } from "../db";
 
 export class TwitterService {
   private client: Scraper;
@@ -25,31 +19,26 @@ export class TwitterService {
     this.twitterUsername = config.username;
   }
 
-  private async setCookiesFromArray(cookiesArray: TwitterCookie[]) {
-    const cookieStrings = cookiesArray.map(
+  private async setCookiesFromArray(cookies: TwitterCookie[]) {
+    const cookieStrings = cookies.map(
       (cookie) =>
-        `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${
-          cookie.secure ? "Secure" : ""
-        }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${
-          cookie.sameSite || "Lax"
+        `${cookie.name}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${cookie.secure ? "Secure" : ""
+        }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${cookie.sameSite || "Lax"
         }`,
     );
     await this.client.setCookies(cookieStrings);
   }
 
   async initialize() {
     try {
-      // Ensure cache directory exists
-      await ensureCacheDirectory();
-
       // Check for cached cookies
-      const cachedCookies = await getCachedCookies(this.twitterUsername);
+      const cachedCookies = db.getTwitterCookies(this.twitterUsername);
       if (cachedCookies) {
         await this.setCookiesFromArray(cachedCookies);
       }
 
       // Load last checked tweet ID from cache
-      this.lastCheckedTweetId = await getLastCheckedTweetId();
+      this.lastCheckedTweetId = db.getTwitterCacheValue("last_tweet_id");
 
       // Try to login with retries
       logger.info("Attempting Twitter login...");
@@ -64,7 +53,16 @@ export class TwitterService {
           if (await this.client.isLoggedIn()) {
             // Cache the new cookies
             const cookies = await this.client.getCookies();
-            await cacheCookies(this.config.username, cookies);
+            const formattedCookies = cookies.map(cookie => ({
+              name: cookie.key,
+              value: cookie.value,
+              domain: cookie.domain,
+              path: cookie.path,
+              secure: cookie.secure,
+              httpOnly: cookie.httpOnly,
+              sameSite: cookie.sameSite as "Strict" | "Lax" | "None" | undefined,
+            }));
+            db.setTwitterCookies(this.config.username, formattedCookies);
             break;
           }
         } catch (error) {
@@ -154,7 +152,7 @@ export class TwitterService {
 
   async setLastCheckedTweetId(tweetId: string) {
     this.lastCheckedTweetId = tweetId;
-    await saveLastCheckedTweetId(tweetId);
+    db.setTwitterCacheValue("last_tweet_id", tweetId);
     logger.info(`Last checked tweet ID updated to: ${tweetId}`);
   }
 

backend/src/services/twitter/queries.ts

@@ -0,0 +1,73 @@
+import { eq } from "drizzle-orm";
+import { twitterCache, twitterCookies } from "./schema";
+import { BunSQLiteDatabase } from "drizzle-orm/bun-sqlite";
+
+// Twitter Cookie Management
+export function getTwitterCookies(db: BunSQLiteDatabase, username: string) {
+  return db
+    .select()
+    .from(twitterCookies)
+    .where(eq(twitterCookies.username, username))
+    .get();
+}
+
+export function setTwitterCookies(
+  db: BunSQLiteDatabase,
+  username: string,
+  cookiesJson: string,
+) {
+  return db
+    .insert(twitterCookies)
+    .values({
+      username,
+      cookies: cookiesJson,
+    })
+    .onConflictDoUpdate({
+      target: twitterCookies.username,
+      set: {
+        cookies: cookiesJson,
+        updatedAt: new Date().toISOString(),
+      },
+    });
+}
+
+export function deleteTwitterCookies(db: BunSQLiteDatabase, username: string) {
+  return db.delete(twitterCookies).where(eq(twitterCookies.username, username));
+}
+
+// Twitter Cache Management
+export function getTwitterCacheValue(db: BunSQLiteDatabase, key: string) {
+  return db
+    .select()
+    .from(twitterCache)
+    .where(eq(twitterCache.key, key))
+    .get();
+}
+
+export function setTwitterCacheValue(
+  db: BunSQLiteDatabase,
+  key: string,
+  value: string,
+) {
+  return db
+    .insert(twitterCache)
+    .values({
+      key,
+      value,
+    })
+    .onConflictDoUpdate({
+      target: twitterCache.key,
+      set: {
+        value,
+        updatedAt: new Date().toISOString(),
+      },
+    });
+}
+
+export function deleteTwitterCacheValue(db: BunSQLiteDatabase, key: string) {
+  return db.delete(twitterCache).where(eq(twitterCache.key, key));
+}
+
+export function clearTwitterCache(db: BunSQLiteDatabase) {
+  return db.delete(twitterCache);
+}

backend/src/services/twitter/schema.ts

@@ -0,0 +1,27 @@
+import { sqliteTable as table, text } from "drizzle-orm/sqlite-core";
+
+// Reusable timestamp columns
+const timestamps = {
+  createdAt: text("created_at")
+    .notNull()
+    .$defaultFn(() => new Date().toISOString()),
+  updatedAt: text("updated_at").$defaultFn(() => new Date().toISOString()),
+};
+
+export const twitterCookies = table(
+  "twitter_cookies",
+  {
+    username: text("username").primaryKey(),
+    cookies: text("cookies").notNull(), // JSON string of TwitterCookie[]
+    ...timestamps,
+  }
+);
+
+export const twitterCache = table(
+  "twitter_cache",
+  {
+    key: text("key").primaryKey(), // e.g., "last_tweet_id"
+    value: text("value").notNull(),
+    ...timestamps,
+  }
+);

backend/src/types/twitter.ts

@@ -25,3 +25,14 @@ export interface TwitterConfig {
   password: string;
   email: string;
 }
+
+export interface TwitterCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  expires?: number;
+  httpOnly?: boolean;
+  secure?: boolean;
+  sameSite?: "Strict" | "Lax" | "None";
+}