diff --git a/Dockerfile b/Dockerfile index 19ce761..c751a6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,7 @@ COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs RUN mkdir -p /litefs /var/lib/litefs /public && \ chown -R bun:bun /litefs /var/lib/litefs /public +# Create volume mount points # Copy only necessary files from builders COPY --from=backend-builder --chown=bun:bun /app/package.json ./ COPY --chown=bun:bun curate.config.json ./ @@ -60,7 +61,6 @@ COPY --from=backend-builder --chown=bun:bun /app/backend/dist ./backend/dist # Set environment variables ENV DATABASE_URL="file:/litefs/db" -ENV CACHE_DIR="/litefs/cache" ENV NODE_ENV="production" # Expose the port diff --git a/backend/src/services/db/index.ts b/backend/src/services/db/index.ts index 45974fe..014e1a3 100644 --- a/backend/src/services/db/index.ts +++ b/backend/src/services/db/index.ts @@ -1,11 +1,15 @@ import { Database } from "bun:sqlite"; import { BunSQLiteDatabase, drizzle } from "drizzle-orm/bun-sqlite"; import { join } from "node:path"; -import { Moderation, TwitterSubmission } from "types/twitter"; + +import { logger } from "utils/logger"; import { broadcastUpdate } from "../../index"; + import * as queries from "./queries"; -import { logger } from "utils/logger"; +// Twitter +import { Moderation, TwitterSubmission, TwitterCookie } from "types/twitter"; +import * as twitterQueries from "../twitter/queries"; export class DatabaseService { private db: BunSQLiteDatabase; private static readonly DB_PATH = @@ -121,6 +125,46 @@ export class DatabaseService { // For now, content is the same as submission since we're dealing with tweets return this.getSubmission(contentId); } + + // Twitter Cookie Management + setTwitterCookies(username: string, cookies: TwitterCookie[]): void { + const cookiesJson = JSON.stringify(cookies); + twitterQueries.setTwitterCookies(this.db, username, cookiesJson).run(); + } + + getTwitterCookies(username: string): TwitterCookie[] | null { + const result = twitterQueries.getTwitterCookies(this.db, username); + if (!result) return null; + + try { + return JSON.parse(result.cookies) as TwitterCookie[]; + } catch (e) { + logger.error("Error parsing Twitter cookies:", e); + return null; + } + } + + deleteTwitterCookies(username: string): void { + twitterQueries.deleteTwitterCookies(this.db, username).run(); + } + + // Twitter Cache Management + setTwitterCacheValue(key: string, value: string): void { + twitterQueries.setTwitterCacheValue(this.db, key, value).run(); + } + + getTwitterCacheValue(key: string): string | null { + const result = twitterQueries.getTwitterCacheValue(this.db, key); + return result?.value ?? null; + } + + deleteTwitterCacheValue(key: string): void { + twitterQueries.deleteTwitterCacheValue(this.db, key).run(); + } + + clearTwitterCache(): void { + twitterQueries.clearTwitterCache(this.db).run(); + } } // Export a singleton instance diff --git a/backend/src/services/db/queries.ts b/backend/src/services/db/queries.ts index 140a9d9..db65494 100644 --- a/backend/src/services/db/queries.ts +++ b/backend/src/services/db/queries.ts @@ -1,13 +1,13 @@ import { and, eq, sql } from "drizzle-orm"; import { BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"; +import { Moderation, TwitterSubmission } from "types/twitter"; import { + feeds, moderationHistory, submissionCounts, - submissions, - feeds, submissionFeeds, + submissions } from "./schema"; -import { Moderation, TwitterSubmission } from "types/twitter"; export function upsertFeed( db: BunSQLiteDatabase, @@ -172,11 +172,11 @@ export function getSubmission( submittedAt: result.submittedAt, moderationHistory: result.moderationHistory ? JSON.parse(`[${result.moderationHistory}]`) - .filter((m: any) => m !== null) - .map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], }; } @@ -223,11 +223,11 @@ export function getSubmissionByAcknowledgmentTweetId( submittedAt: result.submittedAt, moderationHistory: result.moderationHistory ? JSON.parse(`[${result.moderationHistory}]`) - .filter((m: any) => m !== null) - .map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], }; } @@ -268,11 +268,11 @@ export function getAllSubmissions(db: BunSQLiteDatabase): TwitterSubmission[] { submittedAt: result.submittedAt, moderationHistory: result.moderationHistory ? JSON.parse(result.moderationHistory) - .filter((m: any) => m !== null) - .map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], })); } @@ -317,11 +317,11 @@ export function getSubmissionsByStatus( submittedAt: result.submittedAt, moderationHistory: result.moderationHistory ? JSON.parse(result.moderationHistory) - .filter((m: any) => m !== null) - .map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], })); } @@ -452,11 +452,11 @@ export function getSubmissionsByFeed( submittedAt: result.submittedAt, moderationHistory: result.moderationHistory ? JSON.parse(result.moderationHistory) - .filter((m: any) => m !== null) - .map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], })); } diff --git a/backend/src/services/db/schema.ts b/backend/src/services/db/schema.ts index ec37371..5102124 100644 --- a/backend/src/services/db/schema.ts +++ b/backend/src/services/db/schema.ts @@ -6,6 +6,9 @@ import { text, } from "drizzle-orm/sqlite-core"; +// From exports/plugins +export * from "../twitter/schema"; + // Reusable timestamp columns const timestamps = { createdAt: text("created_at") diff --git a/backend/src/services/twitter/client.ts b/backend/src/services/twitter/client.ts index ad790fc..e8a0949 100644 --- a/backend/src/services/twitter/client.ts +++ b/backend/src/services/twitter/client.ts @@ -1,13 +1,7 @@ import { Scraper, SearchMode, Tweet } from "agent-twitter-client"; +import { TwitterCookie } from "types/twitter"; import { logger } from "../../utils/logger"; -import { - TwitterCookie, - cacheCookies, - ensureCacheDirectory, - getCachedCookies, - getLastCheckedTweetId, - saveLastCheckedTweetId, -} from "../../utils/cache"; +import { db } from "../db"; export class TwitterService { private client: Scraper; @@ -25,13 +19,11 @@ export class TwitterService { this.twitterUsername = config.username; } - private async setCookiesFromArray(cookiesArray: TwitterCookie[]) { - const cookieStrings = cookiesArray.map( + private async setCookiesFromArray(cookies: TwitterCookie[]) { + const cookieStrings = cookies.map( (cookie) => - `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${ - cookie.secure ? "Secure" : "" - }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${ - cookie.sameSite || "Lax" + `${cookie.name}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}; ${cookie.secure ? "Secure" : "" + }; ${cookie.httpOnly ? "HttpOnly" : ""}; SameSite=${cookie.sameSite || "Lax" }`, ); await this.client.setCookies(cookieStrings); @@ -39,17 +31,14 @@ export class TwitterService { async initialize() { try { - // Ensure cache directory exists - await ensureCacheDirectory(); - // Check for cached cookies - const cachedCookies = await getCachedCookies(this.twitterUsername); + const cachedCookies = db.getTwitterCookies(this.twitterUsername); if (cachedCookies) { await this.setCookiesFromArray(cachedCookies); } // Load last checked tweet ID from cache - this.lastCheckedTweetId = await getLastCheckedTweetId(); + this.lastCheckedTweetId = db.getTwitterCacheValue("last_tweet_id"); // Try to login with retries logger.info("Attempting Twitter login..."); @@ -64,7 +53,16 @@ export class TwitterService { if (await this.client.isLoggedIn()) { // Cache the new cookies const cookies = await this.client.getCookies(); - await cacheCookies(this.config.username, cookies); + const formattedCookies = cookies.map(cookie => ({ + name: cookie.key, + value: cookie.value, + domain: cookie.domain, + path: cookie.path, + secure: cookie.secure, + httpOnly: cookie.httpOnly, + sameSite: cookie.sameSite as "Strict" | "Lax" | "None" | undefined, + })); + db.setTwitterCookies(this.config.username, formattedCookies); break; } } catch (error) { @@ -154,7 +152,7 @@ export class TwitterService { async setLastCheckedTweetId(tweetId: string) { this.lastCheckedTweetId = tweetId; - await saveLastCheckedTweetId(tweetId); + db.setTwitterCacheValue("last_tweet_id", tweetId); logger.info(`Last checked tweet ID updated to: ${tweetId}`); } diff --git a/backend/src/services/twitter/queries.ts b/backend/src/services/twitter/queries.ts new file mode 100644 index 0000000..b315ce1 --- /dev/null +++ b/backend/src/services/twitter/queries.ts @@ -0,0 +1,73 @@ +import { eq } from "drizzle-orm"; +import { twitterCache, twitterCookies } from "./schema"; +import { BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"; + +// Twitter Cookie Management +export function getTwitterCookies(db: BunSQLiteDatabase, username: string) { + return db + .select() + .from(twitterCookies) + .where(eq(twitterCookies.username, username)) + .get(); +} + +export function setTwitterCookies( + db: BunSQLiteDatabase, + username: string, + cookiesJson: string, +) { + return db + .insert(twitterCookies) + .values({ + username, + cookies: cookiesJson, + }) + .onConflictDoUpdate({ + target: twitterCookies.username, + set: { + cookies: cookiesJson, + updatedAt: new Date().toISOString(), + }, + }); +} + +export function deleteTwitterCookies(db: BunSQLiteDatabase, username: string) { + return db.delete(twitterCookies).where(eq(twitterCookies.username, username)); +} + +// Twitter Cache Management +export function getTwitterCacheValue(db: BunSQLiteDatabase, key: string) { + return db + .select() + .from(twitterCache) + .where(eq(twitterCache.key, key)) + .get(); +} + +export function setTwitterCacheValue( + db: BunSQLiteDatabase, + key: string, + value: string, +) { + return db + .insert(twitterCache) + .values({ + key, + value, + }) + .onConflictDoUpdate({ + target: twitterCache.key, + set: { + value, + updatedAt: new Date().toISOString(), + }, + }); +} + +export function deleteTwitterCacheValue(db: BunSQLiteDatabase, key: string) { + return db.delete(twitterCache).where(eq(twitterCache.key, key)); +} + +export function clearTwitterCache(db: BunSQLiteDatabase) { + return db.delete(twitterCache); +} \ No newline at end of file diff --git a/backend/src/services/twitter/schema.ts b/backend/src/services/twitter/schema.ts new file mode 100644 index 0000000..31ec25d --- /dev/null +++ b/backend/src/services/twitter/schema.ts @@ -0,0 +1,27 @@ +import { sqliteTable as table, text } from "drizzle-orm/sqlite-core"; + +// Reusable timestamp columns +const timestamps = { + createdAt: text("created_at") + .notNull() + .$defaultFn(() => new Date().toISOString()), + updatedAt: text("updated_at").$defaultFn(() => new Date().toISOString()), +}; + +export const twitterCookies = table( + "twitter_cookies", + { + username: text("username").primaryKey(), + cookies: text("cookies").notNull(), // JSON string of TwitterCookie[] + ...timestamps, + } +); + +export const twitterCache = table( + "twitter_cache", + { + key: text("key").primaryKey(), // e.g., "last_tweet_id" + value: text("value").notNull(), + ...timestamps, + } +); diff --git a/backend/src/types/twitter.ts b/backend/src/types/twitter.ts index 10534cd..2517881 100644 --- a/backend/src/types/twitter.ts +++ b/backend/src/types/twitter.ts @@ -25,3 +25,14 @@ export interface TwitterConfig { password: string; email: string; } + +export interface TwitterCookie { + name: string; + value: string; + domain: string; + path: string; + expires?: number; + httpOnly?: boolean; + secure?: boolean; + sameSite?: "Strict" | "Lax" | "None"; +} diff --git a/backend/src/utils/cache.ts b/backend/src/utils/cache.ts deleted file mode 100644 index 6afc5cf..0000000 --- a/backend/src/utils/cache.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { logger } from "./logger"; -import path from "path"; -import fs from "fs/promises"; - -export interface TwitterCookie { - key: string; - value: string; - domain: string; - path: string; - secure: boolean; - httpOnly: boolean; - sameSite?: string; -} - -interface CookieCache { - [username: string]: TwitterCookie[]; -} - -const CACHE_DIR = process.env.CACHE_DIR || ".cache"; - -export async function ensureCacheDirectory() { - try { - try { - await fs.access(CACHE_DIR); - } catch { - // Directory doesn't exist, create it - await fs.mkdir(CACHE_DIR, { recursive: true }); - logger.info("Created cache directory"); - } - } catch (error) { - logger.error("Failed to create cache directory:", error); - throw error; - } -} - -export async function getCachedCookies( - username: string, -): Promise { - try { - // Try to read cookies from a local cache file - const cookiePath = path.join(CACHE_DIR, ".twitter-cookies.json"); - - const data = await fs.readFile(cookiePath, "utf-8"); - const cache: CookieCache = JSON.parse(data); - - if (cache[username]) { - return cache[username]; - } - } catch (error) { - // If file doesn't exist or is invalid, return null - return null; - } - return null; -} - -export async function getLastCheckedTweetId(): Promise { - try { - const tweetPath = path.join(CACHE_DIR, ".last-tweet-id"); - const data = await fs.readFile(tweetPath, "utf-8"); - return data.trim() || null; - } catch (error) { - // If file doesn't exist or is invalid, return null - return null; - } -} - -export async function saveLastCheckedTweetId(tweetId: string) { - try { - const tweetPath = path.join(CACHE_DIR, ".last-tweet-id"); - await fs.writeFile(tweetPath, tweetId); - } catch (error) { - logger.error("Failed to cache last tweet ID:", error); - } -} - -export async function cacheCookies(username: string, cookies: TwitterCookie[]) { - try { - const cookiePath = path.join(CACHE_DIR, ".twitter-cookies.json"); - - let cache: CookieCache = {}; - try { - const data = await fs.readFile(cookiePath, "utf-8"); - cache = JSON.parse(data); - } catch (error) { - // If file doesn't exist, start with empty cache - } - - cache[username] = cookies; - await fs.writeFile(cookiePath, JSON.stringify(cache, null, 2)); - } catch (error) { - logger.error("Failed to cache cookies:", error); - } -} diff --git a/fly.toml b/fly.toml index 6f87282..aa634cd 100644 --- a/fly.toml +++ b/fly.toml @@ -16,6 +16,12 @@ primary_region = 'den' source = "litefs" destination = "/var/lib/litefs" initial_size = '1GB' + +[[mounts]] + source = "public" + destination = "/public" + initial_size = '1GB' + [http_service] internal_port = 3000 force_https = true @@ -28,3 +34,4 @@ primary_region = 'den' memory = '1gb' cpu_kind = 'shared' cpus = 1 + \ No newline at end of file diff --git a/next.txt b/next.txt index 32dd351..75d7339 100644 --- a/next.txt +++ b/next.txt @@ -6,4 +6,9 @@ aside from !submit @twitter_username || @twitter_username !submit, any hashtags submission needs processing (processSubmission) any additional text is the note save submission to feed (need table for feeds) -on moderation, handle approve or reject \ No newline at end of file +on moderation, handle approve or reject + + + +write everything to sqlite +