Merge pull request #5 from PotLock/feat/queue

Adds plugin support and queue management

README.md

@@ -219,9 +219,18 @@ TWITTER_EMAIL=your_twitter_email
 
 It will use these credentials to login and cache cookies via [agent-twitter-client](https://github.com/ai16z/agent-twitter-client).
 
+### Configuration
+
+See the [Configuration Documentation](./docs/docs/developers/configuration.md) for detailed information about:
+
+- Feed configuration and structure
+- Stream and recap behavior
+- Plugin system
+- Message queue handling
+
 ### Admin Configuration
 
-Admins are Twitter accounts that have moderation privileges. Configure admin accounts in `backend/src/config/admins.ts`:
+Admins are Twitter accounts that have moderation privileges (Twitter handles without @). Configure admin accounts in `backend/src/config/admins.ts`:
 
 ```typescript
 export const ADMIN_ACCOUNTS: string[] = [

backend/package.json

@@ -7,8 +7,8 @@
     "build": "bun build ./src/index.ts --target=bun --outdir=dist --format=esm --external './src/external' && cp -r src/external dist/external/",
     "start": "bun run dist/index.js",
     "dev": "bun run --watch src/index.ts",
-    "test": "jest",
-    "test:watch": "jest --watch",
+    "test": "bun test",
+    "test:watch": "bun test --watch",
     "db:generate": "bun drizzle-kit generate",
     "db:migrate": "bun drizzle-kit migrate",
     "db:push": "bun drizzle-kit push",
@@ -37,16 +37,21 @@
     "bun-types": "^1.1.40",
     "drizzle-kit": "^0.30.1",
     "jest": "^29.7.0",
+    "jest-mock-extended": "^4.0.0-beta1",
     "ts-jest": "^29.2.5",
     "ts-node": "^10.9.1",
     "typescript": "^5.3.3"
   },
   "dependencies": {
+    "@elysiajs/cors": "^1.2.0",
+    "@elysiajs/swagger": "^1.2.0",
+    "@libsql/client": "^0.14.0",
     "@types/cors": "^2.8.17",
     "agent-twitter-client": "^0.0.16",
     "cors": "^2.8.5",
     "dotenv": "^16.0.3",
     "drizzle-orm": "^0.38.3",
+    "elysia": "^1.2.10",
     "express": "^4.18.2",
     "ora": "^8.1.1",
     "winston": "^3.17.0",

backend/src/__tests__/mocks/db-service.mock.ts

@@ -0,0 +1,79 @@
+import { mock } from "bun:test";
+import { TwitterSubmission } from "../../types/twitter";
+
+// In-memory storage for mock database
+const storage = {
+  submissions: new Map<string, TwitterSubmission>(),
+  submissionFeeds: new Map<string, Set<string>>(), // tweetId -> Set of feedIds
+  dailySubmissionCounts: new Map<string, number>(), // userId -> count
+  acknowledgments: new Map<string, string>(), // tweetId -> acknowledgmentTweetId
+  moderationResponses: new Map<string, string>(), // tweetId -> responseTweetId
+};
+
+export const mockDb = {
+  upsertFeed: mock<(feed: { id: string; name: string; description?: string }) => void>(() => {}),
+
+  getDailySubmissionCount: mock<(userId: string) => number>((userId) => {
+    return storage.dailySubmissionCounts.get(userId) || 0;
+  }),
+
+  saveSubmission: mock<(submission: TwitterSubmission) => void>((submission) => {
+    storage.submissions.set(submission.tweetId, submission);
+  }),
+
+  saveSubmissionToFeed: mock<(submissionId: string, feedId: string) => void>((submissionId, feedId) => {
+    const feeds = storage.submissionFeeds.get(submissionId) || new Set();
+    feeds.add(feedId);
+    storage.submissionFeeds.set(submissionId, feeds);
+  }),
+
+  incrementDailySubmissionCount: mock<(userId: string) => void>((userId) => {
+    const currentCount = storage.dailySubmissionCounts.get(userId) || 0;
+    storage.dailySubmissionCounts.set(userId, currentCount + 1);
+  }),
+
+  updateSubmissionAcknowledgment: mock<(tweetId: string, acknowledgmentTweetId: string) => void>((tweetId, ackId) => {
+    storage.acknowledgments.set(tweetId, ackId);
+  }),
+
+  getSubmissionByAcknowledgmentTweetId: mock<(acknowledgmentTweetId: string) => TwitterSubmission | null>((ackId) => {
+    for (const [tweetId, storedAckId] of storage.acknowledgments.entries()) {
+      if (storedAckId === ackId) {
+        return storage.submissions.get(tweetId) || null;
+      }
+    }
+    return null;
+  }),
+
+  saveModerationAction: mock<(moderation: any) => void>(() => {}),
+
+  updateSubmissionStatus: mock<(tweetId: string, status: "approved" | "rejected", responseTweetId: string) => void>((tweetId, status, responseId) => {
+    const submission = storage.submissions.get(tweetId);
+    if (submission) {
+      submission.status = status;
+      storage.moderationResponses.set(tweetId, responseId);
+    }
+  }),
+
+  getFeedsBySubmission: mock<(submissionId: string) => Array<{ feedId: string }>>((submissionId) => {
+    const feeds = storage.submissionFeeds.get(submissionId) || new Set();
+    return Array.from(feeds).map(feedId => ({ feedId }));
+  }),
+
+  removeFromSubmissionFeed: mock<(submissionId: string, feedId: string) => void>((submissionId, feedId) => {
+    const feeds = storage.submissionFeeds.get(submissionId);
+    if (feeds) {
+      feeds.delete(feedId);
+    }
+  }),
+};
+
+// Helper to reset all mock functions and storage
+export const resetMockDb = () => {
+  Object.values(mockDb).forEach(mockFn => mockFn.mockReset());
+  storage.submissions.clear();
+  storage.submissionFeeds.clear();
+  storage.dailySubmissionCounts.clear();
+  storage.acknowledgments.clear();
+  storage.moderationResponses.clear();
+};

backend/src/__tests__/mocks/distribution-service.mock.ts

@@ -0,0 +1,19 @@
+export class MockDistributionService {
+  public processedSubmissions: Array<{
+    feedId: string;
+    submissionId: string;
+    content: string;
+  }> = [];
+
+  async processStreamOutput(
+    feedId: string,
+    submissionId: string,
+    content: string
+  ): Promise<void> {
+    this.processedSubmissions.push({ feedId, submissionId, content });
+  }
+
+  async processRecapOutput(): Promise<void> {
+    // Not needed for current tests
+  }
+}

backend/src/__tests__/mocks/drizzle.mock.ts

@@ -0,0 +1,38 @@
+import { mock } from 'bun:test';
+import { TwitterSubmission } from '../../types/twitter';
+
+// Define the database interface to match our schema
+interface DbInterface {
+  upsertFeed: (feed: { id: string; name: string; description?: string }) => void;
+  getDailySubmissionCount: (userId: string) => number;
+  saveSubmission: (submission: TwitterSubmission) => void;
+  saveSubmissionToFeed: (submissionId: string, feedId: string) => void;
+  incrementDailySubmissionCount: (userId: string) => void;
+  updateSubmissionAcknowledgment: (tweetId: string, acknowledgmentTweetId: string) => void;
+  getSubmissionByAcknowledgmentTweetId: (acknowledgmentTweetId: string) => Promise<TwitterSubmission | null>;
+  saveModerationAction: (moderation: any) => void;
+  updateSubmissionStatus: (tweetId: string, status: "approved" | "rejected", responseTweetId: string) => void;
+  getFeedsBySubmission: (submissionId: string) => Promise<Array<{ feedId: string }>>;
+  removeFromSubmissionFeed: (submissionId: string, feedId: string) => void;
+}
+
+// Create mock functions for each database operation
+export const drizzleMock = {
+  upsertFeed: mock<DbInterface['upsertFeed']>(() => {}),
+  getDailySubmissionCount: mock<DbInterface['getDailySubmissionCount']>(() => 0),
+  saveSubmission: mock<DbInterface['saveSubmission']>(() => {}),
+  saveSubmissionToFeed: mock<DbInterface['saveSubmissionToFeed']>(() => {}),
+  incrementDailySubmissionCount: mock<DbInterface['incrementDailySubmissionCount']>(() => {}),
+  updateSubmissionAcknowledgment: mock<DbInterface['updateSubmissionAcknowledgment']>(() => {}),
+  getSubmissionByAcknowledgmentTweetId: mock<DbInterface['getSubmissionByAcknowledgmentTweetId']>(() => Promise.resolve(null)),
+  saveModerationAction: mock<DbInterface['saveModerationAction']>(() => {}),
+  updateSubmissionStatus: mock<DbInterface['updateSubmissionStatus']>(() => {}),
+  getFeedsBySubmission: mock<DbInterface['getFeedsBySubmission']>(() => Promise.resolve([])),
+  removeFromSubmissionFeed: mock<DbInterface['removeFromSubmissionFeed']>(() => {}),
+};
+
+// Mock the db module
+import { db } from '../../services/db';
+Object.assign(db, drizzleMock);
+
+export default drizzleMock;

backend/src/__tests__/mocks/twitter-service.mock.ts

@@ -0,0 +1,66 @@
+import { Tweet } from "agent-twitter-client";
+
+export class MockTwitterService {
+  private lastCheckedId: string | null = null;
+  private mockTweets: Tweet[] = [];
+  private mockUserIds: Map<string, string> = new Map();
+
+  async initialize(): Promise<void> {
+    // Mock implementation - do nothing
+  }
+
+  async login(): Promise<void> {
+    // Mock implementation - do nothing
+  }
+
+  async isLoggedIn(): Promise<boolean> {
+    return true;
+  }
+
+  async getCookies(): Promise<any[]> {
+    return [];
+  }
+
+  async setCookies(): Promise<void> {
+    // Mock implementation - do nothing
+  }
+
+  async getUserIdByScreenName(handle: string): Promise<string> {
+    return this.mockUserIds.get(handle) || "mock-user-id";
+  }
+
+  getLastCheckedTweetId(): string | null {
+    return this.lastCheckedId;
+  }
+
+  async setLastCheckedTweetId(id: string | null): Promise<void> {
+    this.lastCheckedId = id;
+  }
+
+  async replyToTweet(tweetId: string, message: string): Promise<string> {
+    return `reply-${tweetId}`;
+  }
+
+  async getTweet(tweetId: string): Promise<Tweet | null> {
+    return this.mockTweets.find(t => t.id === tweetId) || null;
+  }
+
+  async fetchAllNewMentions(lastCheckedId: string | null): Promise<Tweet[]> {
+    if (!lastCheckedId) return this.mockTweets;
+    const index = this.mockTweets.findIndex(t => t.id === lastCheckedId);
+    if (index === -1) return this.mockTweets;
+    return this.mockTweets.slice(index + 1);
+  }
+
+  addMockTweet(tweet: Tweet) {
+    this.mockTweets.push(tweet);
+  }
+
+  addMockUserId(handle: string, id: string) {
+    this.mockUserIds.set(handle, id);
+  }
+
+  clearMockTweets() {
+    this.mockTweets = [];
+  }
+}