Migrates plugins to module federation (#64)

* Migrates to use module federation for plugins (#61) * Adds module federation support for distributor + transformer plugins (#55) * adds memory bank and plugin loader * adds plugin service * wip * working distributor plugins w/ module federation, some TOODs * working, nice, clean plugin service * fmt * addresses comments * set the correct remotes * update memory bank and documentation * implements transform plugins * update memories * fmt * fmt * fix docs for ai-transform * fmt * remove broken link * fix docs * fmt * passing tests * adds tests * adds multi item error * add necessary dependencies for better-sqlite-3 * remove libsql and copy over frontend dist * install @libsql/client * adds externals * fmt * use node for building * convert to better-sqlite-3 * improve docker image * fmt * Revert "Migrates to use module federation for plugins (#61)" (#62) This reverts commit 6c48325. * Migrates plugins to use module federation (#63) * Adds module federation support for distributor + transformer plugins (#55) * adds memory bank and plugin loader * adds plugin service * wip * working distributor plugins w/ module federation, some TOODs * working, nice, clean plugin service * fmt * addresses comments * set the correct remotes * update memory bank and documentation * implements transform plugins * update memories * fmt * fmt * fix docs for ai-transform * fmt * remove broken link * fix docs * fmt * passing tests * adds tests * adds multi item error * add necessary dependencies for better-sqlite-3 * remove libsql and copy over frontend dist * install @libsql/client * adds externals * fmt * use node for building * convert to better-sqlite-3 * improve docker image * fmt * migrate to hono and node * clean up * nodemon * fmt * replace bun * fix bun command * fix npm command * fix scripts * npx * correct command * node module hoisting * fix packages * fmt * package.json * wip for good deploy * fmt * normalizes db async * fmt * fix dist * working plugins * fmt
PotLock · Feb 27, 2025 · 70426b6 · 70426b6
1 parent 00ee01f
commit 70426b6
Show file tree

Hide file tree

Showing 75 changed files with 7,429 additions and 1,834 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,78 +1,85 @@
-## NOTE
-# This Dockerfile builds the frontend and backend separately,
-# frontend uses npm and backend requires bun.
-# This separation is a temporary solution for a Bun issue with rsbuild,
-# see: https://github.com/oven-sh/bun/issues/11628 
-
-# Frontend deps & build stage
-FROM node:20 as frontend-builder
+# Build stage
+FROM node:20 AS builder
 WORKDIR /app
 
-# Copy frontend package files
-COPY frontend/package.json ./frontend/
-
-# Install frontend dependencies
-RUN cd frontend && npm install
-
-# Copy frontend source code
-COPY frontend ./frontend
-
-# Build frontend
-RUN cd frontend && npm run build
-
-# Backend deps & build stage
-FROM oven/bun as backend-builder
-WORKDIR /app
+# Install build dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3 \
+    make \
+    g++ \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
 
-# Copy backend package files
+# Copy package files for dependency installation
 COPY package.json ./
+COPY frontend/package.json ./frontend/
 COPY backend/package.json ./backend/
 COPY backend/drizzle.config.ts ./backend/
 
-# Install backend dependencies
-RUN cd backend && bun install
+# Install dependencies
+RUN cd frontend && npm install
+RUN cd backend && npm install
 
-# Copy backend source code
+# Copy source code after dependency installation
+COPY frontend ./frontend
 COPY backend ./backend
+COPY curate.config.json ./
 
+# Build backend (rspack will copy frontend dist to backend/dist/public)
 ENV NODE_ENV="production"
-
-# Build backend
-RUN cd backend && bun run build
+# Build frontend first since backend depends on it
+RUN cd frontend && npm run build
+# Then build backend which will copy frontend dist
+RUN cd backend && npm run build
 
 # Production stage
-FROM oven/bun as production
+FROM node:20-slim AS production
 WORKDIR /app
 
-# Install LiteFS dependencies
-RUN apt-get update -y && apt-get install -y ca-certificates fuse3 sqlite3
+# Install LiteFS and runtime dependencies
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    fuse3 \
+    sqlite3 \
+    python3 \
+    make \
+    g++ \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
 
 # Copy LiteFS binary
 COPY --from=flyio/litefs:0.5 /usr/local/bin/litefs /usr/local/bin/litefs
 
 # Create directories for mounts with correct permissions
 RUN mkdir -p /litefs /var/lib/litefs && \
-    chown -R bun:bun /litefs /var/lib/litefs
+    chown -R node:node /litefs /var/lib/litefs
 
 # Create volume mount points
-# Set environment variables first
 ENV DATABASE_URL="file:/litefs/db"
-ENV FRONTEND_DIST_PATH="/app/frontend/dist"
 
-# Copy only necessary files from builders
-COPY --from=backend-builder --chown=bun:bun /app/package.json ./
-COPY --chown=bun:bun curate.config.json ./
+# Copy application files
+COPY --from=builder --chown=node:node /app/backend/dist ./backend/dist
+COPY --from=builder --chown=node:node /app/backend/package.json ./backend/package.json
+COPY --from=builder --chown=node:node /app/backend/drizzle.config.ts ./backend/drizzle.config.ts
+COPY --from=builder --chown=node:node /app/backend/src ./backend/src
+COPY --chown=node:node curate.config.json ./
+COPY --chown=node:node package.json ./
 
-COPY --from=frontend-builder --chown=bun:bun /app/frontend/dist ./frontend/dist
-COPY --from=backend-builder --chown=bun:bun /app/backend ./backend
+# Install production dependencies
+RUN cd backend && npm install && npm rebuild better-sqlite3
 
-RUN cd backend && bun install
+# Copy LiteFS configuration
+COPY --chown=node:node litefs.yml /etc/litefs.yml
 
 # Expose the port
 EXPOSE 3000
 
-# Copy LiteFS configuration
-COPY --chown=bun:bun litefs.yml /etc/litefs.yml
+# Set secure environment defaults
+ENV NODE_ENV=production \
+    NPM_CONFIG_LOGLEVEL=warn
 
 # Start LiteFS (runs app with distributed file system for SQLite)
 ENTRYPOINT ["litefs", "mount"]
diff --git a/README.md b/README.md
@@ -120,8 +120,8 @@ bun run dev
 
 This will launch:
 
-- Frontend at http://localhost:5173
-- Backend at http://localhost:3000
+- Frontend at <http://localhost:5173>
+- Backend at <http://localhost:3000>
 
 ### Building for production
 

diff --git a/backend/bun.lock b/backend/bun.lock
diff --git a/backend/bun.lockb b/backend/bun.lockb
diff --git a/backend/package.json b/backend/package.json
@@ -1,21 +1,19 @@
 {
   "name": "@curatedotfun/backend",
   "version": "0.0.1",
-  "packageManager": "[email protected]",
-  "type": "module",
   "scripts": {
-    "build": "bun build ./src/index.ts --target=bun --outdir=dist --format=esm --external './src/external' && cp -r src/external dist/external/",
-    "start": "bun run dist/index.js",
-    "dev": "bun run --watch src/index.ts",
+    "build": "rspack build",
+    "start": "node dist/main.js",
+    "dev": "NODE_ENV=development nodemon --exec \"rspack build && node dist/main.js\" --watch src",
     "test": "bun test",
-    "test:watch": "bun test --watch",
-    "db:generate": "bun drizzle-kit generate",
-    "db:migrate": "bun drizzle-kit migrate",
-    "db:push": "bun drizzle-kit push",
-    "db:pull": "bun drizzle-kit pull",
-    "db:check": "bun drizzle-kit check",
-    "db:up": "bun drizzle-kit up",
-    "db:studio": "bun drizzle-kit studio"
+    "test:watch": "npm test --watch",
+    "db:generate": "drizzle-kit generate",
+    "db:migrate": "drizzle-kit migrate",
+    "db:push": "drizzle-kit push",
+    "db:pull": "drizzle-kit pull",
+    "db:check": "drizzle-kit check",
+    "db:up": "drizzle-kit up",
+    "db:studio": "drizzle-kit studio"
   },
   "browserslist": {
     "production": [
@@ -30,29 +28,33 @@
     ]
   },
   "devDependencies": {
+    "@curatedotfun/types": "^0.0.5",
+    "@module-federation/node": "^2.6.22",
+    "@rspack/cli": "latest",
+    "@types/better-sqlite3": "^7.6.9",
+    "@types/node": "^20.17.19",
     "@types/ora": "^3.2.0",
-    "bun-types": "^1.1.43",
-    "drizzle-kit": "^0.30.1",
+    "concurrently": "^9.1.2",
     "jest": "^29.7.0",
     "jest-mock-extended": "^4.0.0-beta1",
+    "nodemon": "^3.1.9",
     "ts-jest": "^29.2.5",
     "ts-node": "^10.9.1",
-    "typescript": "^5.3.3"
+    "typescript": "^5.3.3",
+    "wait-on": "^8.0.2",
+    "zod": "^3.22.4"
   },
   "dependencies": {
-    "@elysiajs/cors": "^1.2.0",
-    "@elysiajs/static": "^1.2.0",
-    "@elysiajs/swagger": "^1.2.0",
-    "@libsql/client": "^0.14.0",
+    "@hono/node-server": "^1.8.2",
+    "@hono/zod-openapi": "^0.9.5",
+    "@hono/zod-validator": "^0.1.11",
     "@notionhq/client": "^2.2.15",
-    "@types/cors": "^2.8.17",
     "agent-twitter-client": "^0.0.16",
-    "cors": "^2.8.5",
-    "dotenv": "^16.0.3",
+    "better-sqlite3": "11.8.1",
+    "dotenv": "^16.4.7",
+    "drizzle-kit": "^0.30.1",
     "drizzle-orm": "^0.38.3",
-    "elysia": "^1.2.10",
-    "elysia-helmet": "^2.0.0",
-    "express": "^4.18.2",
+    "hono": "^4.0.5",
     "ora": "^8.1.1",
     "winston": "^3.17.0",
     "winston-console-format": "^1.0.8"

diff --git a/backend/rspack.config.js b/backend/rspack.config.js
@@ -0,0 +1,59 @@
+require("dotenv").config();
+const path = require("path");
+const { rspack } = require("@rspack/core");
+
+const isProduction = process.env.NODE_ENV === "production";
+
+module.exports = {
+  entry: "./src/index",
+  mode: isProduction ? "production" : "development",
+  target: "async-node",
+  devtool: "source-map",
+  externals: {
+    "better-sqlite3": "commonjs better-sqlite3",
+    bufferutil: "commonjs bufferutil",
+  },
+  output: {
+    path: path.resolve(__dirname, "dist"),
+    clean: true,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: "builtin:swc-loader",
+        exclude: /node_modules/,
+      },
+      {
+        test: /\.md$/,
+        type: "asset/source",
+      },
+    ],
+  },
+  resolve: {
+    extensions: [".tsx", ".ts", ".js"],
+  },
+  plugins: [
+    new rspack.CopyRspackPlugin({
+      patterns: [
+        {
+          from: "../frontend/dist",
+          to: "public",
+          noErrorOnMissing: true, // Don't error in development when dist doesn't exist
+        },
+      ],
+    }),
+    //   new rspack.container.ModuleFederationPlugin({
+    //     name: "host",
+    //     runtimePlugins: [
+    //       require.resolve("@module-federation/node/runtimePlugin"),
+    //     ],
+    //     shared: {
+    //       "@curatedotfun/types": {
+    //         singleton: true,
+    //         eager: true
+    //       },
+    //     }
+    //   })
+  ],
+};