Merge pull request #147 from getmaxun/develop

chore: release v0.0.2

.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+npm-debug.log
+dist
+.git
+.gitignore
+.env
+.md
+.vscode
+coverage
+docker-compose.yml
+Dockerfile

ENVEXAMPLE

@@ -1,12 +1,12 @@
 # App Setup
 NODE_ENV=production                     # Set to 'development' or 'production' as required
-JWT_SECRET=your_jwt_secret_key          # Replace with a secure JWT secret key
+JWT_SECRET=a9Z$kLq7^f03GzNw!bP9dH4xV6sT2yXl3O8vR@uYq3          # Replace with a secure JWT secret key
 DB_NAME=maxun                           # Your PostgreSQL database name
 DB_USER=postgres                        # PostgreSQL username
 DB_PASSWORD=postgres                    # PostgreSQL password
 DB_HOST=postgres                        # Host for PostgreSQL in Docker
 DB_PORT=5432                            # Port for PostgreSQL (default: 5432)
-ENCRYPTION_KEY=your_encryption_key      # Key for encrypting sensitive data (passwords and proxies)
+ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0      # Key for encrypting sensitive data (passwords and proxies)
 MINIO_ENDPOINT=minio                    # MinIO endpoint in Docker
 MINIO_PORT=9000                         # Port for MinIO (default: 9000)
 MINIO_ACCESS_KEY=minio_access_key       # MinIO access key

README.md

@@ -20,12 +20,17 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web
     <a href="https://x.com/maxun_io"><b>Twitter</b></a> |
     <a href="https://docs.google.com/forms/d/e/1FAIpQLSdbD2uhqC4sbg4eLZ9qrFbyrfkXZ2XsI6dQ0USRCQNZNn5pzg/viewform"><b>Join Maxun Cloud</b></a> | 
     <a href="https://www.youtube.com/@MaxunOSS"><b>Watch Tutorials</b></a>
+    <br />
+    <br />
+<a href="https://trendshift.io/repositories/12113" target="_blank"><img src="https://trendshift.io/api/badge/repositories/12113" alt="getmaxun%2Fmaxun | Trendshift" style="width: 250px; height: 55px; margin-top: 10px;" width="250" height="55"/></a>
 </p>
 
 ![maxun_demo](https://github.com/user-attachments/assets/a61ba670-e56a-4ae1-9681-0b4bd6ba9cdc)
 
 <img src="https://static.scarf.sh/a.png?x-pxid=c12a77cc-855e-4602-8a0f-614b2d0da56a" />
 
+> Note: We are in early stages of development and do not support self hosting yet. You can run Maxun locally.
+
 # Local Setup
 ### Docker Compose
 ```
@@ -49,6 +54,16 @@ npm install
 cd maxun-core 
 npm install
 
+# get back to the root directory
+cd ..
+
+# make sure playwright is properly initialized
+npx playwright install
+npx playwright install-deps
+
+# get back to the root directory
+cd ..
+
 # start frontend and backend together
 npm run start
 ```
@@ -61,8 +76,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
 
 | Variable              | Mandatory | Description                                                                                  | If Not Set                                                   |
 |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------|
-| `BACKEND_URL`            | Yes       | URL to run backend on.                                   | Backend won't start. If not sure, set to http://localhost:8080 |
-| `VITE_BACKEND_URL`            | Yes       | URL to run backend on.                                   | Backend won't start. If not sure, set to http://localhost:8080 |
+| `BACKEND_URL`            | Yes       | URL to run backend on.                                                                    | Default value: http://localhost:8080 |
+| `VITE_BACKEND_URL`            | Yes       | URL used by frontend to connect to backend                                           | Default value: http://localhost:8080 |
 | `JWT_SECRET`          | Yes       | Secret key used to sign and verify JSON Web Tokens (JWTs) for authentication.                | JWT authentication will not work.                            |
 | `DB_NAME`             | Yes       | Name of the Postgres database to connect to.                                                 | Database connection will fail.                               |
 | `DB_USER`             | Yes       | Username for Postgres database authentication.                                               | Database connection will fail.                               |

docker-compose.yml

@@ -32,16 +32,18 @@ services:
     environment:
       MINIO_ROOT_USER: ${MINIO_ACCESS_KEY}
       MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY}
-    command: server /data
+    command: server /data --console-address :9001
     ports:
-      - "9000:9000"
+      - "9000:9000"  # API port
+      - "9001:9001"  # WebUI port
     volumes:
       - minio_data:/data
 
   backend:
-    build:
-      context: .
-      dockerfile: server/Dockerfile
+    #build:
+      #context: .
+      #dockerfile: server/Dockerfile
+    image: getmaxun/maxun-backend:v0.0.1
     ports:
       - "8080:8080"
     env_file: .env
@@ -56,6 +58,7 @@ services:
     - seccomp=unconfined  # This might help with browser sandbox issues
     # Increase shared memory size for Chromium
     shm_size: '2gb'
+    mem_limit: 2g  # Set a 2GB memory limit
     depends_on:
       - postgres
       - redis
@@ -66,9 +69,10 @@ services:
       - /var/run/dbus:/var/run/dbus
 
   frontend:
-    build:
-      context: .
-      dockerfile: Dockerfile
+      #build:
+      #context: .
+      #dockerfile: Dockerfile
+    image: getmaxun/maxun-frontend:v0.0.1
     ports:
       - "5173:5173"
     env_file: .env

package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "author": "Maxun",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
@@ -39,17 +39,19 @@
     "ioredis": "^5.4.1",
     "joi": "^17.6.0",
     "jsonwebtoken": "^9.0.2",
+    "jwt-decode": "^4.0.0",
     "loglevel": "^1.8.0",
     "loglevel-plugin-remote": "^0.6.8",
     "maxun-core": "^0.0.3",
     "minio": "^8.0.1",
     "moment-timezone": "^0.5.45",
     "node-cron": "^3.0.3",
     "pg": "^8.13.0",
-    "playwright": "^1.20.1",
+    "playwright": "^1.48.2",
     "playwright-extra": "^4.3.6",
     "posthog-node": "^4.2.1",
     "prismjs": "^1.28.0",
+    "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
@@ -89,6 +91,7 @@
   "devDependencies": {
     "@types/cookie-parser": "^1.4.7",
     "@types/express": "^4.17.13",
+    "@types/js-cookie": "^3.0.6",
     "@types/loglevel": "^1.6.3",
     "@types/node": "22.7.9",
     "@types/node-cron": "^3.0.11",
@@ -102,6 +105,7 @@
     "ajv": "^8.8.2",
     "concurrently": "^7.0.0",
     "cross-env": "^7.0.3",
+    "js-cookie": "^3.0.5",
     "nodemon": "^2.0.15",
     "ts-node": "^10.4.0",
     "vite": "^5.4.10"

server/Dockerfile

@@ -18,33 +18,6 @@ RUN npm install
 # Install Playwright browsers and dependencies
 RUN npx playwright install --with-deps chromium
 
-# Install xvfb for display support
-#RUN apt-get update && apt-get install -y xvfb
-
-# RUN apt-get update && apt-get install -y \
-#     libgbm-dev \
-#     libxkbcommon-x11-0 \
-#     libxcomposite1 \
-#     libxdamage1 \
-#     libxrandr2 \
-#     libxshmfence1 \
-#     libxtst6 \
-#     libnss3 \
-#     libatk1.0-0 \
-#     libatk-bridge2.0-0 \
-#     libdrm2 \
-#     libxcb1 \
-#     libxkbcommon0 \
-#     fonts-noto-color-emoji \
-#     fonts-unifont \
-#     libpango-1.0-0 \    
-#     libcairo2 \         
-#     libasound2 \        
-#     libglib2.0-0 \      
-#     libdbus-1-3 \       
-#     && rm -rf /var/lib/apt/lists/*
-
-# Create and set permissions for chrome directories
 # Create the Chromium data directory with necessary permissions
 RUN mkdir -p /tmp/chromium-data-dir && \
     chmod -R 777 /tmp/chromium-data-dir

server/src/api/record.ts

@@ -1,6 +1,7 @@
 import { readFile, readFiles } from "../workflow-management/storage";
 import { Router, Request, Response } from 'express';
-import { chromium } from "playwright";
+import { chromium } from "playwright-extra";
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { requireAPIKey } from "../middlewares/api";
 import Robot from "../models/Robot";
 import Run from "../models/Run";
@@ -14,6 +15,7 @@ import { io, Socket } from "socket.io-client";
 import { BinaryOutputService } from "../storage/mino";
 import { AuthenticatedRequest } from "../routes/record"
 import {capture} from "../utils/analytics";
+chromium.use(stealthPlugin());
 
 const formatRecording = (recordingData: any) => {
     const recordingMeta = recordingData.recording_meta;
@@ -289,7 +291,7 @@ router.get("/robots/:id", requireAPIKey, async (req: Request, res: Response) =>
  *                   type: string
  *                   example: "Failed to retrieve runs"
  */
-router.get("/robots/:id/runs", requireAPIKey, async (req: Request, res: Response) => {
+router.get("/robots/:id/runs",requireAPIKey, async (req: Request, res: Response) => {
     try {
         const runs = await Run.findAll({
             where: {
@@ -321,6 +323,7 @@ router.get("/robots/:id/runs", requireAPIKey, async (req: Request, res: Response
 }
 );
 
+
 function formatRunResponse(run: any) {
     const formattedRun = {
         id: run.id,
@@ -494,7 +497,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
     } catch (e) {
         const { message } = e as Error;
         logger.log('info', `Error while scheduling a run with id: ${id}`);
-        console.log(message);
+        console.log(`Error scheduling run:`, message);
         return {
             success: false,
             error: message,
@@ -766,7 +769,6 @@ router.post("/robots/:id/runs", requireAPIKey, async (req: AuthenticatedRequest,
             return res.status(401).json({ ok: false, error: 'Unauthorized' });
         }
         const runId = await handleRunRecording(req.params.id, req.user.dataValues.id);
-        console.log(`Result`, runId);
 
         if (!runId) {
             throw new Error('Run ID is undefined');

server/src/browser-management/classes/RemoteBrowser.ts

@@ -3,9 +3,10 @@ import {
     Browser,
     CDPSession,
     BrowserContext,
-    chromium,
 } from 'playwright';
 import { Socket } from "socket.io";
+import { chromium } from 'playwright-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { PlaywrightBlocker } from '@cliqz/adblocker-playwright';
 import fetch from 'cross-fetch';
 
@@ -14,7 +15,7 @@ import { InterpreterSettings, RemoteBrowserOptions } from "../../types";
 import { WorkflowGenerator } from "../../workflow-management/classes/Generator";
 import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter";
 import { getDecryptedProxyConfig } from '../../routes/proxy';
-
+chromium.use(stealthPlugin());
 
 
 /**
@@ -163,9 +164,7 @@ export class RemoteBrowser {
 
         contextOptions.userAgent = browserUserAgent;
         this.context = await this.browser.newContext(contextOptions);
-        console.log(`Context from initialize: ${JSON.stringify(this.context)}`)
         this.currentPage = await this.context.newPage();
-        console.log(`CPage from initialize: ${JSON.stringify(this.currentPage)}`)
         // await this.currentPage.setExtraHTTPHeaders({
         //     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
         // });
@@ -307,6 +306,7 @@ export class RemoteBrowser {
             const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile());
             await this.initializeNewPage();
             if (this.currentPage) {
+                this.currentPage.setViewportSize({ height: 400, width: 900 });
                 const params = this.generator.getParams();
                 if (params) {
                     this.interpreterSettings.params = params.reduce((acc, param) => {

server/src/browser-management/inputHandlers.ts

@@ -271,7 +271,6 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st
         try {
             await page.goto(url);
             logger.log('debug', `Went to ${url}`);
-            console.log(`Went to ${url}`)
         } catch (e) {
             const { message } = e as Error;
             logger.log('error', message);

server/src/middlewares/auth.ts

@@ -16,12 +16,16 @@ export const requireSignIn = (req: UserRequest, res: Response, next: any) => {
     }
 
     verify(token, secret, (err: any, user: any) => {
-        console.log(err)
-
-        if (err) return res.sendStatus(403)
-
+        if (err) {
+            console.log('JWT verification error:', err);
+            return res.sendStatus(403);
+        }
+        // Normalize payload key
+        if (user.userId && !user.id) {
+            user.id = user.userId;
+            delete user.userId; // temporary: del the old key for clarity
+        }
         req.user = user;
-
-        next()
-    })
+        next();
+    });
 };