diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c4c1c8..938b0f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## [Unreleased] + +### Documentation + +- **Architecture Decision Records.** New [`docs/adr/`](./docs/adr/README.md) + directory captures the eight non-obvious decisions that shaped the + product: Cloudflare-native edge, SearXNG aggregation, Tavily-compat + surface, Workers AI tiered models, Apache 2.0 + self-hostable, + monorepo layout, Python SDK sync+async, honest feature-status policy. +- **`docs/README.md`** is the new doc entry point β€” task-oriented + ("I want to…") rather than alphabetical. +- **`docs/what-is-what.md`** maps every top-level directory and the + meaning of recurring terms (DO, KV, D1, tier, namespace, etc.). +- **`docs/architecture.md`** rewritten to reflect the v2.0 Cloudflare- + native architecture; old "58 endpoints / 27,500 LOC" snapshot replaced + with accurate per-layer detail, data-model table, and end-to-end + request-lifecycle examples. +- **Root docs collapsed into `docs/`.** Deleted seven stale or + duplicate root-level markdown files (`DEPLOYMENT.md`, + `DEPLOYMENT_GUIDE.md`, `DOCKER-COMPOSE-README.md`, + `ENV_VARIABLES.md`, `IMPLEMENTATION_SUMMARY.md`, + `stripe_webhook_setup.md`, `webhook_events_explained.md`). The + canonical versions live under `docs/configuration/`, `docs/deployment/`, + and the new `docs/README.md` index. + +### Added + +- **Python SDK** (`pip install unsearch`) shipping as `apps/sdk-py/`. + Sync + async clients, full Tavily-compat drop-in, SSE streaming, + TypedDict types, py.typed marker. CI matrix across Python 3.9–3.13. + See [ADR-0007](./docs/adr/0007-python-sdk-sync-and-async.md). + ## [Unreleased] β€” Cloudflare-native release Big-bang rewrite from a self-hosted FastAPI/Postgres/Redis/Celery stack diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md deleted file mode 100644 index 90039ba..0000000 --- a/DEPLOYMENT.md +++ /dev/null @@ -1,617 +0,0 @@ -# UnSearch API Deployment Guide - -This guide covers various deployment options for the UnSearch API, from development to production environments. - -## πŸš€ Quick Start (Development) - -### Prerequisites - -- Python 3.11+ -- Redis server -- PostgreSQL database -- SearXNG instance - -### Setup - -```bash -# Clone and setup -git clone -cd UnSearch -make setup - -# Configure environment -cp .env.example .env -# Edit .env with your settings - -# Start services -make docker-up - -# Run migrations -make migrate - -# Start development server -make dev -``` - -## 🐳 Docker Deployment - -### Local Development with Docker - -```bash -# Start all services -docker-compose up -d - -# Check status -docker-compose ps - -# View logs -docker-compose logs -f api - -# Stop services -docker-compose down -``` - -### Production Docker Setup - -1. **Build production image:** - -```bash -make docker-build -``` - -2. **Configure production environment:** - -Create `docker-compose.prod.yml`: - -```yaml -version: "3.8" -services: - api: - image: ghcr.io/rakesh1002/unsearch:latest - environment: - - ENVIRONMENT=production - - DEBUG=false - - API_KEYS=${API_KEYS} - - DATABASE_URL=${DATABASE_URL} - - REDIS_URL=${REDIS_URL} - - SEARXNG_URL=${SEARXNG_URL} - ports: - - "8000:8000" - restart: unless-stopped - depends_on: - - postgres - - redis - - searxng - - worker: - image: ghcr.io/rakesh1002/unsearch:latest - command: celery -A app.workers.tasks worker --loglevel=info - environment: - - ENVIRONMENT=production - - DATABASE_URL=${DATABASE_URL} - - REDIS_URL=${REDIS_URL} - restart: unless-stopped - depends_on: - - postgres - - redis - - flower: - image: ghcr.io/rakesh1002/unsearch:latest - command: celery -A app.workers.tasks flower - ports: - - "5555:5555" - environment: - - REDIS_URL=${REDIS_URL} - restart: unless-stopped - depends_on: - - redis - - postgres: - image: postgres:15-alpine - environment: - - POSTGRES_DB=UnSearch - - POSTGRES_USER=${DB_USER} - - POSTGRES_PASSWORD=${DB_PASSWORD} - volumes: - - postgres_data:/var/lib/postgresql/data - restart: unless-stopped - - redis: - image: redis:7-alpine - command: redis-server --appendonly yes - volumes: - - redis_data:/data - restart: unless-stopped - - searxng: - image: searxng/searxng:latest - ports: - - "8080:8080" - volumes: - - ./searxng:/etc/searxng:rw - restart: unless-stopped - - nginx: - image: nginx:alpine - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - ./ssl:/etc/nginx/ssl:ro - depends_on: - - api - restart: unless-stopped - -volumes: - postgres_data: - redis_data: -``` - -3. **Deploy:** - -```bash -docker-compose -f docker-compose.prod.yml up -d -``` - -## ☸️ Kubernetes Deployment - -### Prerequisites - -- Kubernetes cluster -- kubectl configured -- Helm (optional) - -### 1. Create Namespace - -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: UnSearch -``` - -### 2. ConfigMap and Secrets - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: UnSearch-config - namespace: UnSearch -data: - ENVIRONMENT: "production" - DEBUG: "false" - SEARXNG_URL: "http://searxng:8080" - REDIS_URL: "redis://redis:6379" - ---- -apiVersion: v1 -kind: Secret -metadata: - name: UnSearch-secrets - namespace: UnSearch -type: Opaque -data: - DATABASE_URL: - API_KEYS: -``` - -### 3. Deployment - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: UnSearch-api - namespace: UnSearch -spec: - replicas: 3 - selector: - matchLabels: - app: UnSearch-api - template: - metadata: - labels: - app: UnSearch-api - spec: - containers: - - name: api - image: ghcr.io/rakesh1002/unsearch:latest - ports: - - containerPort: 8000 - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: UnSearch-secrets - key: DATABASE_URL - - name: API_KEYS - valueFrom: - secretKeyRef: - name: UnSearch-secrets - key: API_KEYS - envFrom: - - configMapRef: - name: UnSearch-config - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "1Gi" - cpu: "1000m" - livenessProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /health - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 5 -``` - -### 4. Service and Ingress - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: UnSearch-api-service - namespace: UnSearch -spec: - selector: - app: UnSearch-api - ports: - - port: 80 - targetPort: 8000 - type: ClusterIP - ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: UnSearch-ingress - namespace: UnSearch - annotations: - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod -spec: - tls: - - hosts: - - api.unsearch.dev - secretName: UnSearch-tls - rules: - - host: api.unsearch.dev - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: UnSearch-api-service - port: - number: 80 -``` - -## 🌐 Cloud Deployments - -### AWS ECS - -1. **Create task definition:** - -```json -{ - "family": "UnSearch-api", - "networkMode": "awsvpc", - "requiresCompatibilities": ["FARGATE"], - "cpu": "1024", - "memory": "2048", - "executionRoleArn": "arn:aws:iam::account:role/ecsTaskExecutionRole", - "taskRoleArn": "arn:aws:iam::account:role/ecsTaskRole", - "containerDefinitions": [ - { - "name": "UnSearch-api", - "image": "your-account.dkr.ecr.region.amazonaws.com/UnSearch-api:latest", - "portMappings": [ - { - "containerPort": 8000, - "protocol": "tcp" - } - ], - "environment": [ - { - "name": "ENVIRONMENT", - "value": "production" - } - ], - "secrets": [ - { - "name": "DATABASE_URL", - "valueFrom": "arn:aws:ssm:region:account:parameter/UnSearch/database-url" - } - ], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/UnSearch-api", - "awslogs-region": "us-west-2", - "awslogs-stream-prefix": "ecs" - } - } - } - ] -} -``` - -2. **Create service:** - -```bash -aws ecs create-service \ - --cluster UnSearch-cluster \ - --service-name UnSearch-api \ - --task-definition UnSearch-api:1 \ - --desired-count 2 \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[subnet-xxx],securityGroups=[sg-xxx],assignPublicIp=ENABLED}" -``` - -### Google Cloud Run - -```bash -# Build and push image -gcloud builds submit --tag gcr.io/PROJECT-ID/UnSearch-api - -# Deploy -gcloud run deploy UnSearch-api \ - --image gcr.io/PROJECT-ID/UnSearch-api \ - --platform managed \ - --region us-central1 \ - --allow-unauthenticated \ - --set-env-vars="ENVIRONMENT=production" \ - --set-secrets="DATABASE_URL=projects/PROJECT-ID/secrets/database-url:latest" -``` - -### Azure Container Instances - -```bash -# Create resource group -az group create --name UnSearch-rg --location eastus - -# Deploy container -az container create \ - --resource-group UnSearch-rg \ - --name UnSearch-api \ - --image ghcr.io/rakesh1002/unsearch:latest \ - --dns-name-label UnSearch-api \ - --ports 8000 \ - --environment-variables 'ENVIRONMENT'='production' \ - --secure-environment-variables 'DATABASE_URL'='your-database-url' -``` - -## πŸ”§ Production Configuration - -### Environment Variables - -Key production settings: - -```bash -# Security -ENVIRONMENT=production -DEBUG=false -API_KEYS=key1,key2,key3 -ALLOWED_ORIGINS=https://yourdomain.com - -# Performance -WORKERS=4 -SCRAPING_MAX_CONCURRENT=20 -CACHE_DEFAULT_TTL=3600 - -# Database -DATABASE_URL=postgresql://user:pass@host:5432/db -DATABASE_POOL_SIZE=20 -DATABASE_MAX_OVERFLOW=40 - -# Redis -REDIS_URL=redis://host:6379 -REDIS_MAX_CONNECTIONS=50 - -# Rate Limiting -RATE_LIMIT_DEFAULT=100/hour -RATE_LIMIT_BURST=200 - -# Monitoring -ENABLE_METRICS=true -LOG_LEVEL=INFO -LOG_FORMAT=json -``` - -### SSL/TLS Configuration - -For production, always use HTTPS: - -1. **Nginx SSL configuration:** - -```nginx -server { - listen 443 ssl http2; - server_name api.unsearch.dev; - - ssl_certificate /etc/nginx/ssl/cert.pem; - ssl_certificate_key /etc/nginx/ssl/key.pem; - - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384; - ssl_prefer_server_ciphers off; - - location / { - proxy_pass http://UnSearch-api:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} -``` - -2. **Let's Encrypt with Certbot:** - -```bash -# Install certbot -sudo apt-get install certbot python3-certbot-nginx - -# Get certificate -sudo certbot --nginx -d api.unsearch.dev - -# Auto-renewal -sudo crontab -e -# Add: 0 12 * * * /usr/bin/certbot renew --quiet -``` - -## πŸ“Š Monitoring & Observability - -### Prometheus Metrics - -Configure Prometheus to scrape metrics: - -```yaml -scrape_configs: - - job_name: "UnSearch-api" - static_configs: - - targets: ["api.unsearch.dev:8000"] - metrics_path: "/metrics" - scrape_interval: 15s -``` - -### Grafana Dashboard - -Import the provided Grafana dashboard configuration for monitoring: - -- Request rates and response times -- Error rates by endpoint -- Cache hit rates -- Service health status -- Resource utilization - -### Log Aggregation - -For centralized logging, configure your deployment to send logs to: - -- **ELK Stack** (Elasticsearch, Logstash, Kibana) -- **AWS CloudWatch** -- **Google Cloud Logging** -- **Azure Monitor** - -## πŸ”„ CI/CD Pipeline - -### GitHub Actions Example - -```yaml -name: Deploy UnSearch API - -on: - push: - branches: [main] - release: - types: [published] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install -r requirements.txt - - name: Run tests - run: | - make test-coverage - - build-and-deploy: - needs: test - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v3 - - name: Build Docker image - run: | - docker build -t ghcr.io/rakesh1002/unsearch:${{ github.sha }} . - docker tag ghcr.io/rakesh1002/unsearch:${{ github.sha }} ghcr.io/rakesh1002/unsearch:latest - - name: Deploy to production - run: | - # Your deployment commands here - echo "Deploying to production..." -``` - -## πŸ”’ Security Checklist - -- [ ] Use HTTPS everywhere -- [ ] Configure API keys -- [ ] Set up rate limiting -- [ ] Enable CORS protection -- [ ] Configure security headers -- [ ] Use strong database passwords -- [ ] Enable firewall rules -- [ ] Regular security updates -- [ ] Monitor for vulnerabilities -- [ ] Backup strategy in place - -## 🚨 Troubleshooting - -### Common Issues - -1. **API not responding:** - - - Check service logs: `make logs` - - Verify health endpoint: `curl http://localhost:8000/health` - - Check database connectivity - -2. **High response times:** - - - Monitor cache hit rates - - Check SearXNG performance - - Review concurrent request limits - -3. **Memory issues:** - - - Monitor container resources - - Check for memory leaks - - Adjust worker counts - -4. **Database connection errors:** - - Verify connection string - - Check database server status - - Review connection pool settings - -### Health Checks - -Use the built-in monitoring script: - -```bash -# Single health check -./scripts/monitor.sh - -# Continuous monitoring -./scripts/monitor.sh http://api.unsearch.dev 30 monitor -``` - -## πŸ“ž Support - -For deployment issues: - -1. Check the logs first -2. Review the health endpoints -3. Consult the troubleshooting section -4. Create an issue with detailed information - ---- - -This deployment guide should cover most production scenarios. Adjust configurations based on your specific requirements and infrastructure. diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md deleted file mode 100644 index 2f680d0..0000000 --- a/DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,468 +0,0 @@ -# Complete Deployment Guide - -## Prerequisites - -- GitHub account with repository -- Railway account (https://railway.app) -- Vercel account (https://vercel.com) -- Stripe account (https://stripe.com) - for billing features -- Domain name (optional, but recommended) - -## Part 1: Deploy Backend to Railway - -### Step 1: Prepare Railway Project - -1. **Sign up/Login to Railway**: https://railway.app - -2. **Create New Project**: - - ```bash - # Install Railway CLI - npm install -g @railway/cli - - # Login to Railway - railway login - - # Initialize project in backend directory - cd apps/backend - railway init - ``` - -3. **Connect GitHub Repository**: - - In Railway dashboard, click "New Project" - - Select "Deploy from GitHub repo" - - Choose your repository - - Set root directory to `apps/backend` - -### Step 2: Add Railway Services - -1. **Add PostgreSQL**: - - Click "New" β†’ "Database" β†’ "PostgreSQL" - - Railway automatically provides `DATABASE_URL` - -2. **Add Redis**: - - Click "New" β†’ "Database" β†’ "Redis" - - Railway automatically provides `REDIS_URL` - -### Step 3: Configure Environment Variables - -In Railway dashboard, go to your service β†’ "Variables" and add: - -```bash -# Auto-provided by Railway -DATABASE_URL= -REDIS_URL= - -# Security (REQUIRED - Generate these!) -SECRET_KEY= -JWT_SECRET_KEY= - -# Basic Configuration -ENVIRONMENT=production -DEBUG=false -APP_NAME=UnSearch API -VERSION=1.0.0 -API_PREFIX=/api/v1 - -# CORS (Update with your frontend URL) -ALLOWED_ORIGINS=["https://your-app.vercel.app"] -CORS_CREDENTIALS=true - -# Stripe (Get from Stripe Dashboard) -STRIPE_SECRET_KEY=sk_live_... -STRIPE_PUBLISHABLE_KEY=pk_live_... -STRIPE_WEBHOOK_SECRET=whsec_... -STRIPE_PRO_PRICE_ID=price_... - -# SearXNG (if not using Docker) -SEARXNG_URL=http://searxng-service.railway.internal:8080 - -# Rate Limiting -RATE_LIMIT_ENABLED=true -RATE_LIMIT_DEFAULT=100/minute -``` - -### Step 4: Deploy Backend - -1. **Create railway.json** in `apps/backend/`: - -```json -{ - "$schema": "https://railway.app/railway.schema.json", - "build": { - "builder": "DOCKERFILE", - "dockerfilePath": "./Dockerfile" - }, - "deploy": { - "startCommand": "uvicorn app.main:app --host 0.0.0.0 --port $PORT", - "healthcheckPath": "/health", - "healthcheckTimeout": 100, - "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10 - } -} -``` - -2. **Update Dockerfile** in `apps/backend/`: - -```dockerfile -FROM python:3.11-slim - -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - gcc \ - postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -# Copy requirements -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# Copy application -COPY . . - -# Run migrations on startup -RUN chmod +x scripts/entrypoint.sh - -# Expose port -EXPOSE $PORT - -# Start application -CMD ["sh", "-c", "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port $PORT"] -``` - -3. **Deploy**: - -```bash -# Deploy via CLI -railway up - -# Or push to GitHub (auto-deploys if connected) -git add . -git commit -m "Deploy backend to Railway" -git push origin main -``` - -### Step 5: Get Your Backend URL - -After deployment, Railway provides a URL like: - -- `https://your-app-name.railway.app` - -Save this URL for frontend configuration! - -## Part 2: Deploy Frontend to Vercel - -### Step 1: Prepare Vercel Project - -1. **Install Vercel CLI**: - -```bash -npm install -g vercel -``` - -2. **Login to Vercel**: - -```bash -vercel login -``` - -### Step 2: Configure Frontend - -1. **Update `apps/web/.env.production`**: - -```bash -NEXT_PUBLIC_API_URL=https://your-backend.railway.app -NEXT_PUBLIC_APP_URL=https://your-app.vercel.app -``` - -2. **Ensure `vercel.json` exists in root**: - -```json -{ - "buildCommand": "cd apps/web && npm run build", - "outputDirectory": "apps/web/.next", - "installCommand": "npm install", - "framework": "nextjs", - "regions": ["iad1"], - "env": { - "NEXT_PUBLIC_API_URL": "@backend_url", - "NEXT_PUBLIC_APP_URL": "@frontend_url" - } -} -``` - -### Step 3: Deploy to Vercel - -**Option A: Via CLI** - -```bash -# From repository root -vercel - -# Follow prompts: -# - Set up and deploy? Yes -# - Which scope? Your account -# - Link to existing project? No -# - Project name? unsearch-web -# - Directory? ./apps/web -# - Override settings? No - -# For production deployment -vercel --prod -``` - -**Option B: Via GitHub Integration** - -1. Go to https://vercel.com/new -2. Import your GitHub repository -3. Configure: - - Framework Preset: Next.js - - Root Directory: `apps/web` - - Build Command: `npm run build` - - Install Command: `npm install` -4. Add environment variables: - - `NEXT_PUBLIC_API_URL`: Your Railway backend URL - - `NEXT_PUBLIC_APP_URL`: Your Vercel URL -5. Click "Deploy" - -### Step 4: Configure Environment Variables in Vercel - -In Vercel Dashboard β†’ Settings β†’ Environment Variables: - -```bash -# Production -NEXT_PUBLIC_API_URL=https://your-backend.railway.app -NEXT_PUBLIC_APP_URL=https://your-app.vercel.app - -# Optional -NEXT_PUBLIC_GA_ID=G-XXXXXXXXXX -NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_live_... -``` - -## Part 3: Post-Deployment Setup - -### 1. Configure Custom Domains - -**Railway (Backend)**: - -1. Go to Settings β†’ Domains -2. Add custom domain: `api.yourdomain.com` -3. Update DNS records as instructed - -**Vercel (Frontend)**: - -1. Go to Settings β†’ Domains -2. Add custom domain: `app.yourdomain.com` -3. Update DNS records as instructed - -### 2. Set Up Stripe Webhooks - -1. Go to Stripe Dashboard β†’ Webhooks -2. Add endpoint: `https://api.yourdomain.com/api/v1/billing/webhook` -3. Select events: - - `checkout.session.completed` - - `customer.subscription.created` - - `customer.subscription.updated` - - `customer.subscription.deleted` - - `invoice.payment_succeeded` - - `invoice.payment_failed` -4. Copy webhook secret to Railway environment variables - -### 3. Configure CORS - -Update Railway environment variable: - -```bash -ALLOWED_ORIGINS=["https://app.yourdomain.com","https://yourdomain.com"] -``` - -### 4. Set Up Monitoring - -**Railway Monitoring**: - -- Built-in metrics dashboard -- Set up alerts for high CPU/memory usage - -**Vercel Analytics**: - -- Enable Web Analytics in Vercel dashboard -- Add Speed Insights - -### 5. Database Migrations - -Run migrations after deployment: - -```bash -# Via Railway CLI -railway run alembic upgrade head - -# Or add to deployment command in railway.json -``` - -## Part 4: Deploy SearXNG (Optional) - -If you need your own SearXNG instance: - -### Option A: Deploy to Railway - -1. Create new service in Railway -2. Deploy from Docker image: - -```bash -railway run docker run -d \ - -p 8080:8080 \ - -v searxng:/etc/searxng \ - -e SEARXNG_SECRET=ultrasecretkey \ - searxng/searxng:latest -``` - -### Option B: Use Public Instance - -Update backend environment: - -```bash -SEARXNG_URL=https://searx.be # Or other public instance -``` - -## Deployment Checklist - -### Pre-Deployment - -- [ ] All tests passing -- [ ] Environment variables documented -- [ ] Secrets generated securely -- [ ] Database migrations ready -- [ ] CORS origins configured - -### Backend (Railway) - -- [ ] PostgreSQL database created -- [ ] Redis instance created -- [ ] Environment variables set -- [ ] Deployment successful -- [ ] Health check passing -- [ ] Database migrated - -### Frontend (Vercel) - -- [ ] Environment variables set -- [ ] API URL configured correctly -- [ ] Build successful -- [ ] Deployment successful -- [ ] Can reach backend API - -### Post-Deployment - -- [ ] Custom domains configured -- [ ] SSL certificates active -- [ ] Stripe webhooks configured -- [ ] CORS working properly -- [ ] Monitoring set up -- [ ] Error tracking configured - -## Troubleshooting - -### Backend Issues - -**Database Connection Failed**: - -```bash -# Check DATABASE_URL format -postgresql://user:password@host:port/database?sslmode=require - -# Test connection -railway run python -c "from app.services.database import DatabaseService; import asyncio; asyncio.run(DatabaseService().initialize())" -``` - -**CORS Errors**: - -```python -# Verify ALLOWED_ORIGINS includes your frontend URL -# Check browser console for specific CORS error -``` - -**Port Issues**: - -```bash -# Railway provides PORT env variable -# Ensure using: --port $PORT -``` - -### Frontend Issues - -**API Connection Failed**: - -```javascript -// Check NEXT_PUBLIC_API_URL is set correctly -console.log(process.env.NEXT_PUBLIC_API_URL); - -// Test API endpoint -fetch(`${process.env.NEXT_PUBLIC_API_URL}/health`); -``` - -**Build Failures**: - -```bash -# Clear cache and rebuild -rm -rf .next node_modules -npm install -npm run build -``` - -### Common Commands - -```bash -# Railway CLI -railway logs # View logs -railway run # Run command in production -railway env # List environment variables -railway restart # Restart service - -# Vercel CLI -vercel logs # View logs -vercel env ls # List environment variables -vercel --prod # Deploy to production -vercel rollback # Rollback deployment -``` - -## Cost Estimation - -### Railway - -- **Starter**: $5/month (includes $5 usage) -- **PostgreSQL**: ~$5-10/month -- **Redis**: ~$5/month -- **Total**: ~$15-20/month - -### Vercel - -- **Hobby**: Free (personal use) -- **Pro**: $20/month (commercial use) -- **Bandwidth**: 100GB free, then $0.15/GB - -### Total Monthly Cost - -- **Minimum**: ~$15 (Railway only, Vercel free tier) -- **Recommended**: ~$40 (Railway + Vercel Pro) - -## Security Recommendations - -1. **Enable 2FA** on all accounts (GitHub, Railway, Vercel, Stripe) -2. **Use environment variables** for all secrets -3. **Rotate API keys** every 90 days -4. **Set up alerts** for suspicious activity -5. **Enable audit logs** where available -6. **Use strong passwords** and password managers -7. **Implement rate limiting** on all endpoints -8. **Regular security updates** for dependencies - -## Support Resources - -- **Railway**: https://docs.railway.app -- **Vercel**: https://vercel.com/docs -- **Discord Community**: https://discord.gg/unsearch -- **GitHub Issues**: https://github.com/unsearch/api/issues -- **Email Support**: support@unsearch.dev diff --git a/DOCKER-COMPOSE-README.md b/DOCKER-COMPOSE-README.md deleted file mode 100644 index 8ee3444..0000000 --- a/DOCKER-COMPOSE-README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Docker Compose Files - -## docker-compose.yml -**Purpose:** Full development environment -**Use:** `docker compose up -d` -**Includes:** All services (API, Web, SearXNG, PostgreSQL, Redis, Nginx, Celery) - -## docker-compose.prod.yml -**Purpose:** Production deployment template -**Use:** `docker compose -f docker-compose.prod.yml up -d` -**Optimized:** Smaller images, production settings - -## docker-compose.quickstart.yml -**Purpose:** Quick demo/testing (minimal services) -**Use:** `docker compose -f docker-compose.quickstart.yml up -d` -**Includes:** Only essential services (API, SearXNG, Redis) diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md deleted file mode 100644 index 734065c..0000000 --- a/ENV_VARIABLES.md +++ /dev/null @@ -1,283 +0,0 @@ -# Environment Variables Configuration - -## Backend (FastAPI) - `apps/backend/.env` - -### Required Variables - -```bash -# Database Configuration -DATABASE_URL=postgresql://username:password@localhost:5432/unsearch -# Example: postgresql://postgres:postgres@localhost:5432/unsearch_db - -# Redis Configuration -REDIS_URL=redis://localhost:6379 -# For production with auth: redis://:password@redis-host:6379/0 - -# Security Keys -SECRET_KEY=your-super-secret-key-minimum-32-chars -# Generate: openssl rand -hex 32 -JWT_SECRET_KEY=your-jwt-secret-key-minimum-32-chars -# Generate: openssl rand -hex 32 - -# Environment -ENVIRONMENT=development # Options: development, staging, production -DEBUG=true # Set to false in production - -# API Configuration -APP_NAME=UnSearch API -VERSION=1.0.0 -API_PREFIX=/api/v1 -``` - -### Optional Variables - -```bash -# Stripe Configuration (Required for billing features) -STRIPE_SECRET_KEY=sk_test_... # Test key for development -STRIPE_PUBLISHABLE_KEY=pk_test_... -STRIPE_WEBHOOK_SECRET=whsec_... -STRIPE_PRO_PRICE_ID=price_... # Create in Stripe Dashboard - -# SearXNG Configuration -SEARXNG_URL=http://localhost:8080 # Default if using Docker -SEARXNG_SECRET=change-me-with-openssl-rand-hex-32 # Must match searxng/settings.yml - -# CORS Configuration -ALLOWED_ORIGINS=["http://localhost:3000","https://app.unsearch.dev"] -CORS_CREDENTIALS=true -CORS_METHODS=["GET","POST","PUT","DELETE","OPTIONS"] -CORS_HEADERS=["*"] - -# Rate Limiting -RATE_LIMIT_ENABLED=true -RATE_LIMIT_DEFAULT=100/minute -RATE_LIMIT_STORAGE_URL=redis://localhost:6379/1 - -# Celery Configuration (For async tasks) -CELERY_BROKER_URL=redis://localhost:6379/2 -CELERY_RESULT_BACKEND=redis://localhost:6379/3 - -# Monitoring -PROMETHEUS_ENABLED=true -PROMETHEUS_PORT=9090 -SENTRY_DSN=https://...@sentry.io/... # Optional error tracking - -# Email Configuration (For notifications) -SMTP_HOST=smtp.gmail.com -SMTP_PORT=587 -SMTP_USER=your-email@example.com -SMTP_PASSWORD=your-app-password -SMTP_FROM=noreply@unsearch.dev - -# External Services -OPENAI_API_KEY=sk-... # If using AI features -SLACK_WEBHOOK_URL=https://hooks.slack.com/... # For alerts - -# Performance Tuning -MAX_CONNECTIONS_COUNT=100 -MIN_CONNECTIONS_COUNT=10 -CONNECTION_TIMEOUT=30 -CACHE_TTL=3600 # 1 hour -MAX_CACHE_SIZE=1000 -``` - -## Frontend (Next.js) - `apps/web/.env.local` - -### Required Variables - -```bash -# API Configuration -NEXT_PUBLIC_API_URL=http://localhost:8000 -# Production: https://api.unsearch.dev or your Railway URL - -# App Configuration -NEXT_PUBLIC_APP_URL=http://localhost:3000 -# Production: https://app.unsearch.dev or your Vercel URL -``` - -### Optional Variables - -```bash -# Analytics (Optional) -NEXT_PUBLIC_GA_ID=G-XXXXXXXXXX # Google Analytics -NEXT_PUBLIC_MIXPANEL_TOKEN=your-mixpanel-token -NEXT_PUBLIC_POSTHOG_KEY=your-posthog-key - -# Feature Flags -NEXT_PUBLIC_ENABLE_BILLING=true -NEXT_PUBLIC_ENABLE_DOCS=true -NEXT_PUBLIC_ENABLE_WEBHOOKS=false - -# Stripe (For embedded checkout) -NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_... - -# Sentry (Error tracking) -NEXT_PUBLIC_SENTRY_DSN=https://...@sentry.io/... -SENTRY_AUTH_TOKEN=your-sentry-auth-token -SENTRY_ORG=your-org -SENTRY_PROJECT=your-project - -# Support -NEXT_PUBLIC_SUPPORT_EMAIL=support@unsearch.dev -NEXT_PUBLIC_DISCORD_INVITE=https://discord.gg/... -``` - -## Docker Environment - `docker-compose.yml` - -```yaml -services: - api: - environment: - - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/unsearch - - REDIS_URL=redis://redis:6379 - - SEARXNG_URL=http://searxng:8080 - - SECRET_KEY=${SECRET_KEY} - - JWT_SECRET_KEY=${JWT_SECRET_KEY} - - ENVIRONMENT=production - - postgres: - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=unsearch - - redis: - # No environment variables needed for basic setup - # For production, add: - # command: redis-server --requirepass ${REDIS_PASSWORD} - - searxng: - environment: - - SEARXNG_SECRET=change-me-with-openssl-rand-hex-32 - - SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml -``` - -## Environment Variables by Service - -### PostgreSQL Requirements - -- `DATABASE_URL` must include: - - Username and password - - Host and port - - Database name - - SSL mode for production: `?sslmode=require` - -### Redis Requirements - -- Basic: `redis://host:port` -- With auth: `redis://:password@host:port/db_number` -- Use different DB numbers for different purposes (cache, celery, rate limiting) - -### Stripe Requirements - -1. Create account at https://stripe.com -2. Get API keys from Dashboard -3. Create products and prices -4. Set up webhook endpoint -5. Configure webhook secret - -### SearXNG Requirements - -- Must be accessible from backend -- Settings file must allow API access -- Secret key must match between services - -## Generating Secure Keys - -```bash -# Generate SECRET_KEY and JWT_SECRET_KEY -python -c "import secrets; print(secrets.token_hex(32))" - -# Or using OpenSSL -openssl rand -hex 32 - -# Generate strong passwords -openssl rand -base64 32 -``` - -## Validation Script - -Create `scripts/validate-env.sh`: - -```bash -#!/bin/bash - -# Backend validation -echo "Checking backend environment..." -required_backend=( - "DATABASE_URL" - "REDIS_URL" - "SECRET_KEY" - "JWT_SECRET_KEY" - "ENVIRONMENT" -) - -for var in "${required_backend[@]}"; do - if [ -z "${!var}" ]; then - echo "❌ Missing required variable: $var" - exit 1 - else - echo "βœ… $var is set" - fi -done - -# Frontend validation -echo "Checking frontend environment..." -required_frontend=( - "NEXT_PUBLIC_API_URL" - "NEXT_PUBLIC_APP_URL" -) - -for var in "${required_frontend[@]}"; do - if [ -z "${!var}" ]; then - echo "❌ Missing required variable: $var" - exit 1 - else - echo "βœ… $var is set" - fi -done - -echo "βœ… All required environment variables are set!" -``` - -## Security Best Practices - -1. **Never commit `.env` files** to version control -2. **Use different keys** for different environments -3. **Rotate secrets regularly** (every 90 days) -4. **Use secret management services** in production: - - AWS Secrets Manager - - HashiCorp Vault - - Railway/Vercel environment variables -5. **Encrypt sensitive data** in transit and at rest -6. **Use strong passwords** (minimum 32 characters for keys) -7. **Limit access** to production environment variables - -## Environment-Specific Configurations - -### Development - -```bash -ENVIRONMENT=development -DEBUG=true -DATABASE_URL=postgresql://postgres:postgres@localhost:5432/unsearch_dev -ALLOWED_ORIGINS=["http://localhost:3000"] -``` - -### Staging - -```bash -ENVIRONMENT=staging -DEBUG=false -DATABASE_URL=postgresql://user:pass@staging-db.example.com:5432/unsearch_staging -ALLOWED_ORIGINS=["https://staging.unsearch.dev"] -``` - -### Production - -```bash -ENVIRONMENT=production -DEBUG=false -DATABASE_URL=postgresql://user:pass@prod-db.example.com:5432/unsearch_prod?sslmode=require -ALLOWED_ORIGINS=["https://app.unsearch.dev"] -``` diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index 599e3d3..0000000 --- a/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,350 +0,0 @@ -# UnSearch Platform - Complete Implementation Summary - -## βœ… What Has Been Implemented - -### 1. **Monorepo Architecture** - -- βœ… Migrated to **Turborepo** for efficient monorepo management -- βœ… **Backend** (FastAPI) in `apps/backend/` -- βœ… **Frontend** (Next.js) in `apps/web/` -- βœ… **Shared TypeScript types** in `packages/shared/` -- βœ… All packages building successfully with caching - -### 2. **Backend API** (Existing + Enhanced) - -- βœ… **Authentication System**: JWT-based with refresh tokens -- βœ… **User Management**: Registration, login, profile updates -- βœ… **API Key Management**: Create, list, delete API keys -- βœ… **Billing Integration**: Stripe subscriptions and usage tracking -- βœ… **Search & Scraping**: SearXNG integration with caching -- βœ… **Rate Limiting**: Plan-based limits -- βœ… **Database**: PostgreSQL with Alembic migrations -- βœ… **Caching**: Redis for performance -- βœ… **Background Tasks**: Celery for async processing - -### 3. **Frontend Dashboard** (New) - -- βœ… **Authentication UI**: Login/Register pages with validation -- βœ… **Protected Routes**: HOC for auth-required pages -- βœ… **User Dashboard**: Account overview and quick actions -- βœ… **API Key Management**: - - Create new keys with custom names/descriptions - - View/hide/copy API keys - - Delete keys with confirmation - - Track usage statistics -- βœ… **Responsive Design**: Mobile-friendly with Tailwind CSS -- βœ… **Dark Mode Support**: Theme switching capability -- βœ… **Type Safety**: Full TypeScript with shared types -- βœ… **State Management**: React Query for API state -- βœ… **Error Handling**: Toast notifications with Sonner - -### 4. **Documentation** - -- βœ… **Mintlify Setup**: Professional API documentation -- βœ… **Environment Variables**: Complete guide in `ENV_VARIABLES.md` -- βœ… **Deployment Guide**: Step-by-step for Railway + Vercel -- βœ… **API Examples**: Quickstart and usage examples -- βœ… **README**: Comprehensive project documentation - -### 5. **DevOps & Deployment** - -- βœ… **CI/CD Pipeline**: GitHub Actions workflow -- βœ… **Vercel Configuration**: Frontend deployment ready -- βœ… **Railway Configuration**: Backend deployment ready -- βœ… **Docker Support**: Containerization for all services -- βœ… **Environment Management**: Example files provided -- βœ… **Health Checks**: Monitoring endpoints -- βœ… **Build Optimization**: Turborepo caching - -### 6. **Testing & Verification** - -- βœ… **End-to-End Test Script**: `scripts/test-e2e.sh` -- βœ… **Setup Verification**: `scripts/verify-setup.sh` -- βœ… **Build Tests**: All packages build successfully -- βœ… **Type Checking**: TypeScript validation passing - -## πŸ“ Project Structure - -``` -unsearch/ -β”œβ”€β”€ apps/ -β”‚ β”œβ”€β”€ backend/ # FastAPI application -β”‚ β”‚ β”œβ”€β”€ app/ # Application code -β”‚ β”‚ β”œβ”€β”€ alembic/ # Database migrations -β”‚ β”‚ β”œβ”€β”€ tests/ # Test suites -β”‚ β”‚ β”œβ”€β”€ docker-compose.yml -β”‚ β”‚ β”œβ”€β”€ Dockerfile -β”‚ β”‚ β”œβ”€β”€ requirements.txt -β”‚ β”‚ └── env.example # Environment template -β”‚ └── web/ # Next.js frontend -β”‚ β”œβ”€β”€ src/ -β”‚ β”‚ β”œβ”€β”€ app/ # Pages (App Router) -β”‚ β”‚ β”œβ”€β”€ components/ # UI components -β”‚ β”‚ └── lib/ # Utilities & hooks -β”‚ β”œβ”€β”€ package.json -β”‚ └── env.local.example # Environment template -β”œβ”€β”€ packages/ -β”‚ └── shared/ # Shared TypeScript types -β”‚ β”œβ”€β”€ src/types/ # Type definitions -β”‚ └── package.json -β”œβ”€β”€ docs/ # Mintlify documentation -β”‚ β”œβ”€β”€ mint.json # Mintlify config -β”‚ β”œβ”€β”€ introduction.mdx -β”‚ └── quickstart.mdx -β”œβ”€β”€ .github/ -β”‚ └── workflows/ -β”‚ └── deploy.yml # CI/CD pipeline -β”œβ”€β”€ scripts/ -β”‚ β”œβ”€β”€ test-e2e.sh # End-to-end tests -β”‚ └── verify-setup.sh # Setup verification -β”œβ”€β”€ turbo.json # Turborepo config -β”œβ”€β”€ vercel.json # Vercel deployment -β”œβ”€β”€ package.json # Root package -β”œβ”€β”€ README.md -β”œβ”€β”€ ENV_VARIABLES.md # Environment guide -└── DEPLOYMENT_GUIDE.md # Deployment instructions - -``` - -## πŸš€ Quick Start - -### 1. **Install Dependencies** - -```bash -npm install -``` - -### 2. **Set Up Environment Variables** - -Backend (`apps/backend/.env`): - -```bash -cp apps/backend/env.example apps/backend/.env -# Edit .env with your database, Redis, and API keys -``` - -Frontend (`apps/web/.env.local`): - -```bash -cp apps/web/env.local.example apps/web/.env.local -# Edit .env.local with your API URL -``` - -### 3. **Start Services** - -```bash -# Start everything (backend + frontend) -npm run dev - -# Or start individually: -npm run dev --workspace=apps/backend # Backend on :8000 -npm run dev --workspace=apps/web # Frontend on :3000 -``` - -### 4. **Access Applications** - -- **Frontend**: http://localhost:3000 -- **Backend API**: http://localhost:8000 -- **API Docs**: http://localhost:8000/docs - -## 🚒 Deployment - -### Deploy Backend to Railway - -1. **Push to GitHub** -2. **Create Railway project**: https://railway.app -3. **Add services**: PostgreSQL, Redis -4. **Deploy from GitHub** with root directory: `apps/backend` -5. **Set environment variables** (see ENV_VARIABLES.md) - -### Deploy Frontend to Vercel - -1. **Push to GitHub** -2. **Import to Vercel**: https://vercel.com/new -3. **Configure**: - - Root Directory: `apps/web` - - Framework: Next.js -4. **Set environment variables**: - - `NEXT_PUBLIC_API_URL`: Your Railway backend URL - -### Alternative: Use Provided Scripts - -```bash -# Deploy backend -railway deploy --service backend - -# Deploy frontend -vercel --prod -``` - -## πŸ§ͺ Testing - -### Verify Setup - -```bash -./scripts/verify-setup.sh -``` - -### Run End-to-End Tests - -```bash -# Start services first -npm run dev - -# In another terminal -./scripts/test-e2e.sh -``` - -### Build All Packages - -```bash -npm run build -``` - -## πŸ“‹ Environment Variables - -### Required for Backend - -- `DATABASE_URL` - PostgreSQL connection string -- `REDIS_URL` - Redis connection string -- `SECRET_KEY` - 32+ character secret -- `JWT_SECRET_KEY` - 32+ character JWT secret - -### Required for Frontend - -- `NEXT_PUBLIC_API_URL` - Backend API URL -- `NEXT_PUBLIC_APP_URL` - Frontend app URL - -### Optional (Billing/Features) - -- Stripe keys for payments -- Analytics tracking IDs -- SMTP for emails -- Sentry for error tracking - -See `ENV_VARIABLES.md` for complete list. - -## 🎯 Features Ready for Production - -### User Features - -- βœ… User registration with email/password -- βœ… Secure login with JWT tokens -- βœ… Dashboard with account overview -- βœ… API key generation and management -- βœ… Usage tracking and statistics - -### Developer Features - -- βœ… RESTful API with OpenAPI docs -- βœ… Multiple search engines via SearXNG -- βœ… Content scraping with BeautifulSoup -- βœ… Batch processing support -- βœ… Rate limiting per plan -- βœ… Webhook support for async operations - -### Platform Features - -- βœ… Subscription management with Stripe -- βœ… Free and Pro plans -- βœ… Usage-based billing -- βœ… Admin dashboard capabilities -- βœ… Health monitoring endpoints - -## πŸ“ Still To Implement (Optional Enhancements) - -1. **Billing UI Components**: - - Subscription upgrade/downgrade flow - - Payment method management - - Invoice history view - -2. **Advanced Dashboard Features**: - - Usage charts and analytics - - Search history - - Webhook management UI - -3. **Documentation Viewer**: - - Embedded API documentation - - Interactive API explorer - - Code examples generator - -4. **Additional Features**: - - Email verification - - Password reset flow - - Two-factor authentication - - Team/organization support - -## πŸ”’ Security Considerations - -1. **Generate new secret keys** before deployment -2. **Use HTTPS** in production -3. **Configure CORS** properly for your domains -4. **Enable rate limiting** to prevent abuse -5. **Set up monitoring** and alerts -6. **Regular dependency updates** -7. **Database backups** strategy - -## πŸ“š Resources - -- **Documentation**: `docs/` directory with Mintlify -- **Environment Setup**: `ENV_VARIABLES.md` -- **Deployment Guide**: `DEPLOYMENT_GUIDE.md` -- **API Examples**: `docs/quickstart.mdx` -- **GitHub Actions**: `.github/workflows/deploy.yml` - -## πŸ’» Technology Stack - -### Backend - -- **FastAPI** - High-performance Python web framework -- **PostgreSQL** - Primary database -- **Redis** - Caching and rate limiting -- **SQLAlchemy** - ORM -- **Alembic** - Database migrations -- **Celery** - Background tasks -- **SearXNG** - Privacy-respecting metasearch -- **BeautifulSoup4** - Web scraping -- **Stripe** - Payment processing - -### Frontend - -- **Next.js 15** - React framework with App Router -- **TypeScript** - Type safety -- **Tailwind CSS** - Utility-first styling -- **shadcn/ui** - Component library -- **React Query** - Server state management -- **React Hook Form** - Form handling -- **Zod** - Schema validation -- **Axios** - HTTP client - -### Infrastructure - -- **Turborepo** - Monorepo management -- **Docker** - Containerization -- **GitHub Actions** - CI/CD -- **Railway** - Backend hosting -- **Vercel** - Frontend hosting -- **Mintlify** - Documentation - -## πŸŽ‰ Conclusion - -The UnSearch platform is now a **production-ready monorepo** with: - -- βœ… Full-stack implementation -- βœ… Modern architecture -- βœ… Type-safe development -- βœ… Scalable infrastructure -- βœ… Professional documentation -- βœ… Deployment automation - -**Ready to deploy!** Follow the deployment guide and you'll have your search API platform live in minutes. - -## Support - -- **Documentation**: See `docs/` directory -- **Issues**: Create GitHub issues -- **Email**: support@unsearch.dev - ---- - -Built with ❀️ for developers who value privacy and efficiency. diff --git a/README.md b/README.md index 3cac166..bfdfcdb 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ REDIS_URL="redis://localhost:6379" DATABASE_URL="postgresql://unsearch:${POSTGRES_PASSWORD:-changeme}@localhost:5432/unsearch" ``` -Stripe billing, SMTP, OAuth, monitoring β€” all documented in [docs/configuration/env-variables.md](./docs/configuration/env-variables.md) and [ENV_VARIABLES.md](./ENV_VARIABLES.md). +Stripe billing, SMTP, OAuth, monitoring β€” all documented in [docs/configuration/env-variables.md](./docs/configuration/env-variables.md). --- diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c35e191 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,78 @@ +# UnSearch documentation + +Start here. This page is the entry point for everyone who isn't sure which file to open. + +## I want to… + +### …understand the product + +| For | Read | +|-----|------| +| The 30-second pitch | [`/README.md`](../README.md) (repo root) | +| Why we built UnSearch and for whom | [`strategy/icp.md`](./strategy/icp.md) and [`strategy/positioning.md`](./strategy/positioning.md) | +| What's shipped vs. in beta vs. planned | [`feature-matrix.md`](./feature-matrix.md) | +| Pricing rationale | [`strategy/pricing.md`](./strategy/pricing.md) | +| Where the company is going | [`roadmap.md`](./roadmap.md) and [`strategy/mrr-plan.md`](./strategy/mrr-plan.md) | + +### …use the API + +| For | Read | +|-----|------| +| 5-minute self-host quickstart | [`quickstart.md`](./quickstart.md) | +| Migrate from Tavily | [`migration/from-tavily.md`](./migration/from-tavily.md) | +| Endpoint contracts | [`API_REFERENCE.md`](./API_REFERENCE.md) (or live OpenAPI at `/docs`) | +| Worked examples per endpoint | [`API_EXAMPLES.md`](./API_EXAMPLES.md) | +| Which AI model runs each request | [`ai-pipeline.md`](./ai-pipeline.md) (and [`ai-quick-reference.md`](./ai-quick-reference.md) for a one-pager) | +| Use the Python SDK | [`/apps/sdk-py/README.md`](../apps/sdk-py/README.md) | +| Use the TypeScript SDK | [`/apps/sdk-ts/README.md`](../apps/sdk-ts/README.md) | +| Use the LlamaIndex retriever | [`/apps/sdk-llamaindex/README.md`](../apps/sdk-llamaindex/README.md) | + +### …contribute to the code + +| For | Read | +|-----|------| +| What's where in the repo | [`what-is-what.md`](./what-is-what.md) | +| How the v2.0 architecture works | [`architecture.md`](./architecture.md) | +| Cloudflare-specific wiring | [`cloudflare-architecture.md`](./cloudflare-architecture.md) and [`/workers/README.md`](../workers/README.md) | +| Why we made each major decision | [`adr/`](./adr/README.md) | +| Repo conventions (testing, commits, naming) | [`/CONTRIBUTING.md`](../CONTRIBUTING.md) and [`/CLAUDE.md`](../CLAUDE.md) | +| What shipped recently | [`/CHANGELOG.md`](../CHANGELOG.md) | + +### …operate UnSearch + +| For | Read | +|-----|------| +| Deploy to Cloudflare | [`/workers/README.md`](../workers/README.md) and [`deployment/quick-reference.md`](./deployment/quick-reference.md) | +| Deploy to Railway | [`deployment/railway.md`](./deployment/railway.md) | +| Deploy to DigitalOcean | [`deployment/digitalocean.md`](./deployment/digitalocean.md) | +| On-call playbooks | [`operations/RUNBOOKS.md`](./operations/RUNBOOKS.md) | +| Observability + dashboards | [`/workers/OBSERVABILITY.md`](../workers/OBSERVABILITY.md) | +| Manage secrets | [`SECRETS_MANAGEMENT.md`](./SECRETS_MANAGEMENT.md) and [`/workers/SECRETS.md`](../workers/SECRETS.md) | +| Configure env vars | [`configuration/env-variables.md`](./configuration/env-variables.md) | +| Set up Stripe billing | [`BILLING_SETUP.md`](./BILLING_SETUP.md), [`configuration/stripe-webhook.md`](./configuration/stripe-webhook.md), [`configuration/webhook-events.md`](./configuration/webhook-events.md) | + +### …sell or support UnSearch + +| For | Read | +|-----|------| +| The ICP definition | [`strategy/icp.md`](./strategy/icp.md) | +| Jobs-to-be-done framework | [`strategy/jtbd.md`](./strategy/jtbd.md) | +| Sales playbook | [`strategy/sales-playbook.md`](./strategy/sales-playbook.md) | +| GTM plan | [`strategy/gtm.md`](./strategy/gtm.md) | +| User journey + activation | [`strategy/user-journey.md`](./strategy/user-journey.md) | +| Market + competitor landscape | [`strategy/market.md`](./strategy/market.md) | +| Value proposition | [`strategy/value-prop.md`](./strategy/value-prop.md) | + +--- + +## Doc conventions + +- **Status taxonomy.** Every feature claim uses βœ… shipped / πŸ”Ά in beta / πŸ“‹ planned. See [ADR-0008](./adr/0008-honest-feature-status-policy.md). +- **Single source of truth.** [`feature-matrix.md`](./feature-matrix.md) is canonical for status; [`CHANGELOG.md`](../CHANGELOG.md) is canonical for what shipped when. Other docs link to these β€” they don't restate. +- **Code/doc co-location.** Per-package READMEs live next to the code: `apps/*/README.md`, `workers/README.md`. Cross-cutting docs live here. +- **No emoji in code or commit messages.** Emoji are fine in docs only. +- **ADRs document non-obvious decisions.** Don't write one for routine implementation choices. See [`adr/README.md`](./adr/README.md). + +## When something is wrong + +If something in these docs is inaccurate, please file an issue at [github.com/Rakesh1002/unsearch/issues](https://github.com/Rakesh1002/unsearch/issues). Documentation rot is real and we'd rather know. diff --git a/docs/adr/0001-cloudflare-native-edge-architecture.md b/docs/adr/0001-cloudflare-native-edge-architecture.md new file mode 100644 index 0000000..24c1d52 --- /dev/null +++ b/docs/adr/0001-cloudflare-native-edge-architecture.md @@ -0,0 +1,45 @@ +# ADR-0001: Cloudflare-native edge architecture + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +UnSearch v1 ran the classic "managed VPS + Postgres + Redis + Celery" stack. Three problems were structural: + +1. **Latency.** Search traffic is globally distributed; an origin-only deploy meant ~200–400ms p95 from outside the origin region. +2. **Cost at the high-traffic ICP.** Persona A β€” indie devs hitting 100K+ searches/mo β€” would burn through a $20/mo VPS in cold-start storms and trigger always-on Celery workers we paid for at idle. +3. **Vendor positioning.** Tavily / Exa / Brave all run their own infra. Our pitch ("10Γ— cheaper, Apache 2.0, self-hostable") needs a unit-economics story that holds at the price point. Cloudflare's free-tier-then-cheap pricing curve maps onto our pricing curve. + +## Decision + +Adopt **Cloudflare-native edge** as the default deploy target: + +- **Cloudflare Workers (Hono router)** front every request at `workers/src/index.ts`. Endpoints that don't need Python (KV cache hits, simple search proxying, auth checks, rate limiting) terminate at the edge. +- **Cloudflare Containers** host the FastAPI origin (`Dockerfile.cloudflare`, `workers/containers.toml`) with auto-scale 0–10. Workloads that need Python's ecosystem (heavy scraping, complex orchestration, alembic migrations against the long-tail of features) proxy from the worker via a service binding. +- **D1** is the primary edge database (`workers/schema.sql`). Postgres remains as the local-dev origin DB and as an escape hatch for ops that need it. +- **KV** for hot-path caches (auth, rate-limit counters, search cache hits). +- **R2** for object storage (scraped HTML snapshots, large extracted artifacts). +- **Vectorize** for the RAG corpus β€” `bge-m3` 1024-dim embeddings. +- **Queues** for async work (research-agent fan-out, monitor checks, batch crawls). +- **Durable Objects** for stateful coordination β€” `RateLimiter` (sliding window), `TopicMonitor` (alarms + webhook fan-out), `ResearchAgent` (multi-step LLM-driven research), `SessionManager` (chat/pagination cursors). +- **Workers AI** for LLM inference and embeddings β€” see [ADR-0004](./0004-workers-ai-tiered-model-selection.md). + +The split between "lives at the edge" and "lives in the Container" is documented in [`docs/cloudflare-architecture.md`](../cloudflare-architecture.md). + +## Consequences + +- **Pro:** Single-vendor edge story. The deploy is `wrangler deploy` for the edge + `wrangler deploy --config workers/containers.toml` for the Container. No Kubernetes, no managed Postgres bill for v2. +- **Pro:** Global p95 latency drops to <100ms for KV-cache hits, <200ms for Worker AI calls. +- **Pro:** Free tier on the platform side maps neatly onto our 5,000-req/mo free tier on the product side. +- **Con:** Self-host story is more complex than "docker compose up" β€” we keep `docker-compose.yml` working as the no-Cloudflare path (see [ADR-0005](./0005-apache-2-license-self-hostable-from-day-one.md)), but it deliberately leaves the edge-resident features (Vectorize, Workers AI) as optional. +- **Con:** Cloudflare Containers is still maturing β€” direct bindings from inside Containers aren't available, so the FastAPI Container talks to D1 / KV / Queues over REST (see `app/services/core/d1_client.py`, `cache_kv.py`, `queue_producer.py`). +- **Con:** Locks us into Cloudflare's specific quirks (Vectorize index size limits, Durable Object eviction semantics, Workers' 50ms CPU-time budget). + +## Alternatives considered + +- **AWS Lambda + DynamoDB + Bedrock.** Rejected β€” multi-vendor cost story is worse, Bedrock model selection lags Workers AI, latency at the edge requires CloudFront in front of Lambda@Edge (extra hop). +- **Vercel + Vercel KV + OpenAI.** Rejected β€” pricing per-invocation is opaque, vendor lock to a single LLM (OpenAI) breaks the "model selection" pitch, no equivalent of Vectorize as a first-party offering. +- **Stay on VPS + Postgres.** Rejected β€” the latency and unit-economics problems were the explicit reason to move. +- **Hybrid: VPS origin + Cloudflare CDN-only.** Considered. Rejected because it leaves the latency problem unsolved for any endpoint that hits the origin (everything except static asset serving). diff --git a/docs/adr/0002-searxng-as-meta-search-aggregator.md b/docs/adr/0002-searxng-as-meta-search-aggregator.md new file mode 100644 index 0000000..b3426a7 --- /dev/null +++ b/docs/adr/0002-searxng-as-meta-search-aggregator.md @@ -0,0 +1,41 @@ +# ADR-0002: SearXNG as the meta-search aggregator + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +UnSearch needs a web-search backbone. The realistic options for "give me ranked results from the public web" all have trade-offs: + +- **Direct provider APIs (Google CSE, Bing Web Search, Brave Search API).** Per-query costs ($3–$5 per 1k), opaque rate limits, and single-engine results. Vendor lock-in directly contradicts our positioning. +- **Build a crawler.** Multi-month project. Not differentiating. Politically and legally fraught (robots.txt, anti-bot signals, geo-blocking). +- **Buy from Tavily / Exa / Brave.** Defeats the whole product premise. +- **SearXNG** β€” open-source, self-hostable, aggregates 70+ engines, normalizes results, respects robots.txt. + +## Decision + +Use **SearXNG as the meta-search aggregation layer** that sits behind the search API. + +- Self-hosted SearXNG container (`searxng/settings.yml`, `docker-compose.yml`) +- The FastAPI backend posts queries to `SEARXNG_URL` (env-driven) +- We aggregate, dedupe, and re-rank the SearXNG response before returning it to the caller +- Engine selection is exposed via the `engines` parameter on `/api/v1/search` + +This decision is what enables the "Multi-Engine Aggregation πŸš€" row in [`docs/feature-matrix.md`](../feature-matrix.md). It's our most defensible technical wedge against single-provider competitors. + +## Consequences + +- **Pro:** Zero per-query cost for the search itself β€” we pay only for the SearXNG host (and for the AI inference layer on top). +- **Pro:** Engine plurality is a feature: callers can ask for `["google", "duckduckgo", "brave"]` and get a deduped union. +- **Pro:** Self-host story is honest β€” the same SearXNG that powers `api.unsearch.dev` runs in the user's `docker compose up`. +- **Con:** SearXNG is community-maintained. Engines break when upstream providers change their HTML. We mitigate with multi-engine fallback in `app/services/search/`, but the failure mode is real. +- **Con:** No SLA on SearXNG itself. We accept this and design for graceful degradation (route around dead engines, expose engine health via `/api/v1/agent/health`). +- **Con:** Some providers (notably Google) actively rate-limit SearXNG IPs. We work around with proxy rotation in production; the open-source self-host docs note this honestly. + +## Alternatives considered + +- **Direct Bing Web Search API.** Cheapest per-query of the closed APIs, but $3 per 1k still murders our $49/100k pricing. Locks us into one engine. +- **Brave Search API.** Decent but expensive at our volume tier, and Brave is itself a competitor at the SDK layer. +- **Apache Solr / Elasticsearch + open crawl dataset (CommonCrawl).** Considered. Rejected β€” index freshness is 30–60 days behind reality, which is fatal for news/AI-research queries. +- **Build a thin Google CSE wrapper.** Rejected β€” CSE is artificially restricted (10 results, no recency control, expensive at scale). diff --git a/docs/adr/0003-tavily-compatible-drop-in-surface.md b/docs/adr/0003-tavily-compatible-drop-in-surface.md new file mode 100644 index 0000000..7359970 --- /dev/null +++ b/docs/adr/0003-tavily-compatible-drop-in-surface.md @@ -0,0 +1,37 @@ +# ADR-0003: Tavily-compatible drop-in API surface + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +The Persona A ICP (indie devs shipping AI agents) is overwhelmingly already on Tavily. The friction of switching search providers is not the API call itself β€” it's the rewrite, the integration tests, the agent prompts written against a specific response shape. Even at 10Γ— cheaper, "rewrite your integration to save money" is a non-starter for someone who already has a paying product on the line. + +Empirically, the most successful low-cost-vendor migrations (Resend ← SendGrid, Supabase ← Firebase, Bun ← Node) all leaned into wire-level compatibility, not "you'll like ours better." + +## Decision + +Expose **Tavily-compatible drop-in endpoints** alongside our native API. + +- `POST /api/v1/agent/search` mirrors Tavily's `search()` request and response shape one-for-one +- `POST /api/v1/agent/extract` mirrors Tavily's `extract()` +- `POST /api/v1/agent/research` is the UnSearch-specific deep-research extension (no Tavily equivalent yet β€” we don't pretend) +- Both Python and TypeScript SDKs expose `tavily_search` / `tavilySearch` methods that hit `/api/v1/agent/search` β€” explicit naming so the migration intent is obvious in the diff + +The native UnSearch surface (`/api/v1/search`, `/api/v1/neural/*`, `/api/v1/rag/*`) exists in parallel and offers strictly richer parameters (engine selection, scraping toggles, model tier selection). New users who don't have Tavily integrations should use the native surface. + +## Consequences + +- **Pro:** The migration story is "change one base URL + one API key." Lower friction than any other vendor swap on the table. +- **Pro:** Side-by-side dual-write becomes trivial. A caller can call both Tavily and UnSearch with the same request body and diff the response β€” we use this internally to validate parity. +- **Pro:** Tavily's docs become *de facto* discovery docs for our product. Anyone learning Tavily learns our API. +- **Con:** We're locked into matching Tavily's response shape even when their schema is awkward. Their `include_answer` flag, for example, conflates "give me an answer" with "rank results by relevance to the question," which is two features. +- **Con:** Whenever Tavily breaks their schema, we have to make a call: track them, fork the schema, or version (`/api/v2/agent/search`). The honest answer is "track them for now; fork at the first instance of a clearly-bad change." +- **Con:** Marketing risk β€” "Tavily-compatible" frames us as the alternative, not the new category leader. We accept this trade for Persona A acquisition; the Persona B / C story leans on the native surface and the Cloudflare-native architecture. + +## Alternatives considered + +- **Native API only.** Rejected β€” the migration ask is too high for the ICP. +- **Match every closed-source competitor's surface.** Considered β€” neural endpoints (Exa-compat) are also matched, but going beyond two means we spend more time chasing schemas than building. Tavily + Exa neural is the cap. +- **Translation layer (proxy that rewrites Tavily-shape requests to UnSearch-shape).** Rejected β€” adds a hop, leaks abstraction (the proxy needs to know about both schemas, our SDK has to know which mode it's in), and doesn't actually reduce migration friction below the wire-level approach. diff --git a/docs/adr/0004-workers-ai-tiered-model-selection.md b/docs/adr/0004-workers-ai-tiered-model-selection.md new file mode 100644 index 0000000..a0f46d8 --- /dev/null +++ b/docs/adr/0004-workers-ai-tiered-model-selection.md @@ -0,0 +1,48 @@ +# ADR-0004: Workers AI with tiered model selection + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +The AI layer needs to do three things β€” answer generation, embeddings, and reranking β€” across a workload that ranges from "give me a 50-token summary" to "synthesize 12 sources into a 2-page research brief." Pricing AI inference is the single biggest cost lever for an LLM-flavored search product. + +Constraints that ruled out the obvious choices: + +- **Single-LLM strategy (OpenAI-only).** Adds vendor lock that contradicts our positioning. Forces a price-passthrough that destroys margin at the Growth tier. +- **Self-hosted LLM (vLLM on GPUs).** Capex is incompatible with a solo-founder budget. Cold-start latency on autoscaling GPUs is fatal for our p95 target. +- **Bring-your-own-key.** Considered for Enterprise β€” but bad as a default because the playground onboarding flow then asks the user for an API key before they've seen any value. + +## Decision + +Use **Cloudflare Workers AI as the default inference provider, with explicit model tiers** exposed via the API. + +The tiers are picked per-request via the `model_tier` parameter (RAG) or `model` parameter (search): + +| Tier | Model | Use case | +|------|-------|----------| +| `fast` | `@cf/meta/llama-3.1-8b-instruct-fast` | Cheap, low-latency answers; default for free tier | +| `balanced` | `@cf/meta/llama-3.3-70b-instruct-fp8-fast` | Default for Growth tier | +| `reasoning` | `@cf/qwen/qwq-32b` | Multi-step reasoning, research agent | +| `production` | `@cf/openai/gpt-oss-120b` | Highest quality, Enterprise | + +Embeddings: `@cf/baai/bge-m3` (1024 dims) β†’ Cloudflare Vectorize. Reranking: `@cf/baai/bge-reranker-base` over the candidate set returned from SearXNG. + +Tier selection is **explicit, not hidden behind "auto-mode"** β€” callers know what they're getting and we can publish per-tier latency / quality / cost benchmarks honestly. + +## Consequences + +- **Pro:** Single vendor for inference simplifies the cost model and unblocks the Cloudflare-native architecture in [ADR-0001](./0001-cloudflare-native-edge-architecture.md). Workers AI bindings work directly from the edge worker β€” no separate inference endpoint to manage. +- **Pro:** Tier selection becomes a product differentiator. Tavily and Exa hide the model behind their API; we expose it. +- **Pro:** We can change the underlying model in a tier without breaking callers, because the tier name is the contract, not the model ID. +- **Con:** Workers AI's model catalog is more limited than OpenAI's / Anthropic's. We don't have a "Claude Sonnet" or "GPT-4o" tier today, and some workloads (long-context summarization >32k tokens) currently degrade. +- **Con:** Latency-per-token on Workers AI is higher than on the frontier provider APIs for the same parameter count. For interactive playground use this is acceptable; for batch workloads our research agent has to lean on parallelism, not raw throughput. +- **Con:** We're betting on Cloudflare expanding Workers AI's model catalog over time. If that bet doesn't pay off, the escape hatch is per-tier provider routing (e.g., `production` tier β†’ Anthropic Bedrock) without breaking the tier contract. + +## Alternatives considered + +- **OpenAI-only with `gpt-4o` / `gpt-4o-mini`.** Rejected β€” vendor lock, opaque cost passthrough, no edge bindings. +- **Anthropic-only with Claude Sonnet / Haiku.** Same problem, same rejection. (We'd still consider Bedrock as a routing target for the `production` tier if Workers AI quality lags.) +- **Per-request "auto" model selection.** Considered. Rejected because it hides cost from callers and makes our pricing harder to predict. We may revisit when we have multi-tier benchmark data. +- **Self-hosted Ollama / vLLM behind a Cloudflare Tunnel.** Rejected β€” capex, ops burden, cold-start latency, GPU spot-instance instability. diff --git a/docs/adr/0005-apache-2-license-self-hostable-from-day-one.md b/docs/adr/0005-apache-2-license-self-hostable-from-day-one.md new file mode 100644 index 0000000..7cc3096 --- /dev/null +++ b/docs/adr/0005-apache-2-license-self-hostable-from-day-one.md @@ -0,0 +1,47 @@ +# ADR-0005: Apache 2.0 license + self-hostable from day one + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +The competitive landscape (Tavily, Exa, Brave) is uniformly closed-source. That's both an obvious differentiation lane and a strategic question with multiple wrong answers: + +- "Source-available, BSL after N years" (the MongoDB / Sentry / Elastic playbook) is good for hostile-clone defense but bad for the developer-trust pitch we lead with. +- "AGPL" closes off the use case of teams who want to embed our SDK in a closed-source product, which is most of the Persona A market. +- "MIT" is permissive but lacks the patent grant we want for enterprise procurement comfort. + +The license choice locks in a lot β€” it determines who can use the code, whether contributors will assign rights, and whether VC fundraising in the future is constrained by the license history. + +## Decision + +License everything under **Apache 2.0** and commit to a **working self-host path on day one**. + +Concretely: + +- All code in the monorepo is Apache-2.0 (`LICENSE` at repo root, restated in every SDK package). +- `docker compose up -d` from a freshly cloned repo produces a working API in <5 minutes, against a free Cloudflare account or no Cloudflare at all. +- Every feature in `docs/feature-matrix.md` marked βœ… works in self-host. Features that require a Cloudflare account (Workers AI tier, Vectorize) are clearly flagged as such. +- The hosted version at `api.unsearch.dev` runs the **same code** as the self-host. No "open-core" / "Enterprise edition" fork β€” the wedge is hosted convenience + the Cloudflare-native edge, not feature gating. +- No CLA. Contributions sit under the Apache-2.0 grant from the moment they merge. + +This decision is the spine of every other one β€” the SDK choices (separate packages, no proprietary protocols), the architecture (self-hostable Containers, not a closed cloud), the pricing (free tier at 5,000 reqs/mo). + +## Consequences + +- **Pro:** Eliminates the "what if you get acquired and shut down the API" objection in Persona A and B sales conversations. +- **Pro:** Apache 2.0's patent grant is what enterprise legal teams want to see. Reduces friction at the Persona C tier where procurement reviews the license. +- **Pro:** Self-hosting is the ultimate price-fairness commitment β€” if the hosted price ever climbs unreasonably, customers can leave. This is also the *honest* answer to the "10Γ— cheaper, but what's the lock-in risk?" question. +- **Pro:** Public source is the strongest possible recruiting signal. We've already seen interns + community PRs. +- **Con:** Hostile competitor can fork the code. We accept this. The moat is the **hosted UX** (billing, dashboard, Cloudflare-native edge deploy, the curated SearXNG engine config) and the **community** (SDKs, integrations, docs), not the code itself. +- **Con:** Some VC firms have soft preferences against permissive-licensed startups. We accept the smaller funnel of sympathetic investors as a non-issue at our stage. +- **Con:** No "Enterprise tier with extra features" β€” the Enterprise tier is "managed-service + SLA + SOC 2," not "you get features the OSS version doesn't have." This constrains future pricing flexibility, which we judge to be the correct trade. + +## Alternatives considered + +- **MIT.** Considered β€” close call. Apache 2.0 won for the patent grant alone. Patent risk for a search API touching multiple LLM patents is non-zero, and the explicit Apache grant pre-empts that conversation. +- **BSL β†’ Apache 2.0 after 4 years (the MongoDB / Sentry / Elastic playbook).** Rejected β€” the audience we want (Persona A indie devs) reads BSL as "they'll change their mind later," which kills the trust pitch. +- **AGPL.** Rejected β€” closes off embedding inside closed-source products, which is the majority Persona A use case. +- **Open-core (Apache OSS + proprietary Enterprise add-ons).** Rejected β€” the maintenance burden of two codebases, plus the eternal "is this feature OSS or paid?" friction. +- **Source-available "Functional Source License" / "FSL."** Considered. Rejected because it's still too new for procurement teams to recognize, defeating one of the main reasons to be OSS at all. diff --git a/docs/adr/0006-monorepo-with-apps-and-workers.md b/docs/adr/0006-monorepo-with-apps-and-workers.md new file mode 100644 index 0000000..ca17949 --- /dev/null +++ b/docs/adr/0006-monorepo-with-apps-and-workers.md @@ -0,0 +1,62 @@ +# ADR-0006: Monorepo layout with `apps/*` and `workers/` + +- Status: Accepted +- Date: 2026-04-15 +- Deciders: @Rakesh1002 + +## Context + +UnSearch ships: + +- A Python FastAPI backend +- A Next.js dashboard +- A Cloudflare Workers edge router (Hono) +- A TypeScript SDK +- A Python SDK +- A LlamaIndex retriever +- (Soon) an MCP server, a LangChain integration, more + +These don't all share a language, but they *do* share types (request/response shapes), test fixtures, and release cadence. A multi-repo layout would mean N CI pipelines, N versions of the same TypeScript types, and a guaranteed drift between SDK packages and the API they call. + +## Decision + +Use a **single Git repository** with this layout: + +``` +unsearch/ +β”œβ”€β”€ app/ # Legacy single-package FastAPI backend (kept as authoritative origin) +β”œβ”€β”€ apps/ # Monorepo packages +β”‚ β”œβ”€β”€ backend/ # Same FastAPI code, packaged for Container deploy +β”‚ β”œβ”€β”€ web/ # Next.js dashboard (on Workers via @opennextjs/cloudflare) +β”‚ β”œβ”€β”€ sdk-ts/ # @unsearch/sdk β€” TypeScript SDK +β”‚ β”œβ”€β”€ sdk-py/ # unsearch β€” Python SDK +β”‚ └── sdk-llamaindex/ # @unsearch/llamaindex β€” LlamaIndex retriever +β”œβ”€β”€ workers/ # Cloudflare Workers edge router + Durable Objects + D1 schema +β”œβ”€β”€ docs/ # All long-form docs (this directory) +β”œβ”€β”€ alembic/ # Postgres migrations (origin DB) +β”œβ”€β”€ searxng/ # SearXNG meta-search engine config +β”œβ”€β”€ monitoring/ # Prometheus + Grafana provisioning +└── docker-compose*.yml # Self-host stacks +``` + +- **JavaScript/TypeScript packages** use **pnpm workspaces** (`pnpm-workspace.yaml`) β€” one lockfile, one `node_modules`, fast install. +- **Python packages** are independent β€” backend uses `requirements.txt` (root) and a Poetry-managed `pyproject.toml` (`apps/backend/`); SDK uses Hatchling (`apps/sdk-py/pyproject.toml`). No shared Python lockfile. +- **CI scoping** β€” each `apps/*/` package has its own workflow under `.github/workflows/` (e.g., `sdk-py.yml`) triggered on `paths:` filters so unrelated changes don't run unrelated CI. + +The `app/` directory at the repo root is the **historic** single-package backend layout from v1. We kept it because it's the directory production currently imports from (`uvicorn app.main:app`) and migrating that import path is a separate change with its own deploy risk. `apps/backend/` is the monorepo-shaped mirror that builds the Docker image β€” same code, different packaging boundary. + +## Consequences + +- **Pro:** A single PR can change the API endpoint shape and the SDK that consumes it. The diff is the proof of consistency. CI runs the matching tests for both. +- **Pro:** New SDK languages (Go, Ruby, MCP server) drop into `apps/sdk-*/` with a clear template (mirror the TS SDK surface β€” see [ADR-0007](./0007-python-sdk-sync-and-async.md)). +- **Pro:** Docs, ADRs, and code live next to each other. `git blame` on an architecture doc lands in the same history as the code it describes. +- **Con:** Repo size grows monotonically. We mitigate with `path:` filters in CI and per-app `.gitignore`s, but `git clone` time is non-trivial for new contributors. +- **Con:** Two backend layouts (`app/` and `apps/backend/`) is confusing for newcomers. The `docs/what-is-what.md` map documents this explicitly. A future PR will collapse them. +- **Con:** Tooling has to be polyglot β€” Python tests with pytest, TypeScript with vitest, the worker with `wrangler dev`. There's no single "run all tests" command; we accept this in exchange for keeping each ecosystem's conventions. + +## Alternatives considered + +- **Multi-repo (one per package).** Rejected β€” would have made the Tavily-compatibility cross-validation (see [ADR-0003](./0003-tavily-compatible-drop-in-surface.md)) impossible to keep in sync, and forced N versions of each TypeScript type. +- **Nx / Turborepo as the build orchestrator.** Considered. Turborepo is currently in use for the JS side (`.turbo/` exists). We deliberately stop short of using it to orchestrate the Python side; Python tooling is mature enough on its own. +- **Git submodules for SDKs.** Rejected outright β€” submodules guarantee broken `git clone --recurse-submodules` UX for at least 30% of contributors. +- **`packages/` instead of `apps/`.** Considered. `apps/` won because it reads as "deployable unit," and the SDK packages are *also* deployable (to PyPI / npm) β€” making `packages/` redundant. diff --git a/docs/adr/0007-python-sdk-sync-and-async.md b/docs/adr/0007-python-sdk-sync-and-async.md new file mode 100644 index 0000000..8bdb38f --- /dev/null +++ b/docs/adr/0007-python-sdk-sync-and-async.md @@ -0,0 +1,50 @@ +# ADR-0007: Python SDK ships sync + async clients + +- Status: Accepted +- Date: 2026-05-28 +- Deciders: @Rakesh1002 + +## Context + +The Python SDK was the next P0 roadmap item (see [`docs/roadmap.md`](../roadmap.md)). Python's HTTP-client ecosystem is split: + +- The Python community is roughly half sync (notebooks, scripts, classic Flask apps) and half async (FastAPI services, Trio/asyncio apps, agent frameworks like LangChain's async path) +- Competitor SDKs: + - `tavily-python` β€” sync only + - `exa-py` β€” sync only with an `async` wrapper that uses threadpools (not real asyncio) + - `openai-python` β€” both, in a single package, sharing the same surface +- Python 3.12+ has improved async ergonomics enough that "sync-only" is increasingly read as "legacy" + +Forcing a choice creates friction: + +- **Sync-only:** Forces async users to wrap every call in `asyncio.to_thread(...)`, breaks idiomatic FastAPI/LangChain async code, and signals "we didn't think about your stack." +- **Async-only:** Hostile to notebook users and scripts. Breaks the "5-minute pip install demo" because every example has to start with `asyncio.run(...)`. + +## Decision + +The `unsearch` Python package exports **both `UnSearch` and `AsyncUnSearch`** with **identical method surfaces**. + +- Both clients share `httpx` as the underlying HTTP layer (sync + async modes of the same library β€” no two networking stacks). +- Both use **TypedDict request/response types** and a `py.typed` marker so downstream projects get full mypy / pyright coverage. +- Method names match the TypeScript SDK in snake_case form (`neural_search`, `tavily_search`, `start_research`). +- Streaming endpoints (`stream_search`, `stream_rag`) return `Iterator[StreamEvent]` (sync) or `AsyncIterator[StreamEvent]` (async). +- Polling helpers (`poll_research`) sleep with the matching primitive (`time.sleep` / `asyncio.sleep`). + +The package supports Python 3.9–3.13, tested in CI across all five versions. + +## Consequences + +- **Pro:** Drop-in for both audiences. The pip-install demo and the FastAPI/LangChain reference snippets both work without wrapping. +- **Pro:** Type signatures, method names, and request shapes match the TypeScript SDK 1:1 β€” onboarding from one to the other is mechanical. +- **Pro:** `httpx` is the single networking dependency. No `requests` + `aiohttp` split, no `requests` legacy quirks. +- **Pro:** Methods exposed as instance methods (not class methods) so each client can be a context manager (`with UnSearch(...) as client:`), which matters because we set a timeout per-client and don't want lingering connections. +- **Con:** ~2Γ— the surface area to maintain. We mitigate by sharing internals (`_request`, `_headers`, `_parse_sse_chunk`) and a single test fixture set that runs against both clients. +- **Con:** Users have to choose which to import. We address this with a top-of-README example that shows both side by side. +- **Con:** Async clients require Python 3.7+ asyncio, which is fine for 3.9+ but rules out hypothetical Python 2 support. Trivially acceptable. + +## Alternatives considered + +- **Sync-only with a `client.async_session()` helper.** Considered β€” modeled after `requests-async`. Rejected β€” the helper would need to wrap every method, doubling the surface anyway. +- **Async-only and tell users to wrap in `asyncio.run`.** Rejected β€” breaks the notebook experience and is what `exa-py`'s "async wrapper" does badly. +- **Two separate packages (`unsearch` for sync, `unsearch-async` for async).** Rejected β€” splits documentation, splits issues, doubles release work. +- **Use the `anyio` portable async layer to expose a single surface that works both ways.** Considered. Rejected β€” anyio is a great library but the runtime branching makes stack traces harder to read, and our use case doesn't need Trio support. diff --git a/docs/adr/0008-honest-feature-status-policy.md b/docs/adr/0008-honest-feature-status-policy.md new file mode 100644 index 0000000..4ce5aa7 --- /dev/null +++ b/docs/adr/0008-honest-feature-status-policy.md @@ -0,0 +1,48 @@ +# ADR-0008: Honest feature-status policy (βœ… / πŸ”Ά / πŸ“‹) + +- Status: Accepted +- Date: 2026-04-20 +- Deciders: @Rakesh1002 + +## Context + +In the v1 README we claimed "Glean parity βœ…" for a feature set that didn't exist (Glean searches inside-company corpora via connectors we haven't built). We claimed Knowledge Graph, Topic Monitoring, Fact Verification, and Deep Research Agent were "Completed" when in fact the code paths existed but coverage, edge cases, and accuracy were still being closed out. + +That kind of marketing-vs-reality gap is the single fastest way to lose Persona A trust. A senior engineer who tries a feature, finds it half-baked, and feels lied to will not come back for v2. + +We need a way to communicate "this exists, but use it with eyes open" that is more useful than ❌ and more honest than βœ…. + +## Decision + +Adopt a **three-level status taxonomy** used uniformly across docs: + +| Symbol | Meaning | Bar | +|--------|---------|-----| +| βœ… | Shipped | Production code path, end-to-end tested, on the public API, in the SDKs, in CHANGELOG. Promises stability. | +| πŸ”Ά | In beta | Code paths exist and respond, but coverage, edge cases, or polish are still being closed out. Visible in the API. May change shape with notice. | +| πŸ“‹ | Planned | On the roadmap, not in the code. Not in the API. | + +Rules: + +1. **`docs/feature-matrix.md` is the single source of truth.** Any other doc (README, roadmap, marketing site) that claims a status must match it. +2. **Every πŸ”Ά row links to the CHANGELOG `[Unreleased] β€” Deferred to follow-up` section** explaining what still needs to land before it ships to βœ…. +3. **Lying upward is the worst sin.** Marking a πŸ”Ά feature as βœ… to make the matrix look better is a fireable mistake (for the founder; for an intern it's a learning moment). +4. **Lying downward is wasteful.** Marking a feature πŸ“‹ because we want to look modest costs us deals; if it works, mark it βœ…. +5. **The CHANGELOG records every transition.** A πŸ”Ά β†’ βœ… promotion lands in a release entry. A βœ… β†’ πŸ”Ά demotion (yes, this happens) lands too β€” with the reason. + +The same policy applies to the roadmap document, sales decks, the homepage, and the public docs site at docs.unsearch.dev. + +## Consequences + +- **Pro:** Persona A trust is the company's most expensive asset. This policy directly protects it. +- **Pro:** It gives the sales conversation a clean line: "Here's what works today. Here's what's in beta β€” try it, file issues, we're hardening it. Here's what we'll build next." Way better than feature-matrix arms-race claims. +- **Pro:** It frames our πŸ”Ά features as *differentiation* rather than as competitor parity. Tavily/Exa/Brave don't have a Knowledge Graph at all β€” ours being πŸ”Ά is still more than zero, and saying so honestly converts better than fake-βœ… would. +- **Con:** Sales-ops and marketing partners sometimes push for βœ… everywhere. We hold the line. +- **Con:** Investors who skim the matrix may discount πŸ”Ά to ❌ mentally. We accept this β€” the right investor for us is one who can read the matrix carefully. + +## Alternatives considered + +- **Binary βœ… / ❌.** Rejected β€” fails for the half-built case, which is most of the interesting roadmap surface. +- **Five-level (alpha / beta / GA / stable / deprecated).** Rejected β€” too many states, no useful product distinction between "alpha" and "beta" at our stage. +- **Per-endpoint maturity badges, with no top-level matrix.** Considered. Rejected β€” sales conversations need a single page to point at; per-endpoint badges fragment that. +- **"Status: experimental" prose without a symbol.** Rejected β€” symbols make the matrix scannable, which is the only way prospects actually read it. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..fe02e20 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,56 @@ +# Architecture Decision Records + +This directory captures the **non-obvious, sticky decisions** that shaped UnSearch β€” the ones a new contributor would otherwise have to reverse-engineer from the code. + +We use a lightweight [MADR](https://adr.github.io/madr/)-style template: + +``` +# ADR-NNNN: Short imperative title + +- Status: Accepted | Superseded by ADR-XXXX | Deprecated +- Date: YYYY-MM-DD +- Deciders: GitHub handles + +## Context +What is the problem we're solving? What constraints apply? + +## Decision +What did we choose? Phrased so future-us can tell whether the current code still matches. + +## Consequences +What does this commit us to? What did we knowingly give up? + +## Alternatives considered +What did we reject and why? +``` + +## When to write an ADR + +Write one when you're about to make a decision that is: + +- **Hard to reverse** β€” picking a primary datastore, choosing a license, picking a wire format +- **Cross-cutting** β€” affects more than one app/package in the monorepo +- **Surprising in retrospect** β€” would make a new contributor ask "why on earth did they do it that way?" + +Don't write one for routine implementation choices (loop vs. recursion, file naming, etc.). + +## Status taxonomy + +- **Accepted** β€” currently in force, code matches. +- **Superseded by ADR-XXXX** β€” the decision was reversed; the new ADR explains why. +- **Deprecated** β€” no longer in force but kept for history. + +When you supersede an ADR, **don't delete the old one** β€” change its status line and link forward. + +## Index + +| # | Title | Status | +|---|-------|--------| +| [0001](./0001-cloudflare-native-edge-architecture.md) | Cloudflare-native edge architecture | Accepted | +| [0002](./0002-searxng-as-meta-search-aggregator.md) | SearXNG as the meta-search aggregator | Accepted | +| [0003](./0003-tavily-compatible-drop-in-surface.md) | Tavily-compatible drop-in API surface | Accepted | +| [0004](./0004-workers-ai-tiered-model-selection.md) | Workers AI with tiered model selection | Accepted | +| [0005](./0005-apache-2-license-self-hostable-from-day-one.md) | Apache 2.0 + self-hostable from day one | Accepted | +| [0006](./0006-monorepo-with-apps-and-workers.md) | Monorepo layout with `apps/*` + `workers/` | Accepted | +| [0007](./0007-python-sdk-sync-and-async.md) | Python SDK ships sync + async clients | Accepted | +| [0008](./0008-honest-feature-status-policy.md) | Honest feature-status policy (βœ… / πŸ”Ά / πŸ“‹) | Accepted | diff --git a/docs/architecture.md b/docs/architecture.md index b1192b1..c2aeb3d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,378 +1,189 @@ -# UnSearch Backend Architecture +# Architecture -## Executive Summary +> **Scope:** This document describes the v2.0 Cloudflare-native architecture currently running in production. For the per-decision rationale behind each piece, see the [ADRs](./adr/README.md). For the directory-by-directory component map, see [what-is-what.md](./what-is-what.md). For the historic v1 (FastAPI-only) architecture, see git history before [`376f886`](https://github.com/Rakesh1002/unsearch/commit/376f886). -UnSearch is an enterprise-grade AI search platform with **58 API endpoints**, **27,500+ lines of service code**, and comprehensive AI integration via Cloudflare Workers AI. The platform provides Tavily-compatible APIs plus advanced features not available in competitors. +## One-paragraph summary + +UnSearch is a search API for AI agents. Requests land on a **Cloudflare Workers** edge router (`workers/`), which either answers from the edge (KV cache hits, simple proxying, auth, rate-limit) or proxies to a **FastAPI Container** (`app/` and `apps/backend/`) for anything that needs Python's ecosystem (heavy scraping, RAG orchestration, alembic migrations, complex Stripe flows). State lives in **D1** (relational), **KV** (hot caches), **R2** (objects), and **Vectorize** (embeddings); async work runs on **Cloudflare Queues**; multi-step stateful workflows run inside **Durable Objects**. LLM inference and embeddings run on **Cloudflare Workers AI** (see [ADR-0004](./adr/0004-workers-ai-tiered-model-selection.md)). Web search aggregation is delegated to a self-hosted **SearXNG** instance (see [ADR-0002](./adr/0002-searxng-as-meta-search-aggregator.md)). A **Next.js dashboard** (`apps/web/`) deployed via `@opennextjs/cloudflare` lives on the same Workers platform. --- -## System Architecture +## Request-flow diagram ``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ UNSEARCH PLATFORM β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ API LAYER (FastAPI) β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ Agent β”‚ β”‚ Search β”‚ β”‚ RAG β”‚ β”‚Enhanced β”‚ β”‚Advanced β”‚ β”‚ Auth β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ API β”‚ β”‚ API β”‚ β”‚ API β”‚ β”‚ API β”‚ β”‚ v2 API β”‚ β”‚ Billing β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (5 eps) β”‚ β”‚ (4 eps) β”‚ β”‚ (8 eps) β”‚ β”‚ (7 eps) β”‚ β”‚(14 eps) β”‚ β”‚(10 eps) β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ SERVICE LAYER β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ AI β”‚ β”‚ Scraping β”‚ β”‚ Extraction β”‚ β”‚ Crawling β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (3 files) β”‚ β”‚ (9 files) β”‚ β”‚ (9 files) β”‚ β”‚ (8 files) β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ 1,368 LOC β”‚ β”‚ 4,676 LOC β”‚ β”‚ 5,000+ LOC β”‚ β”‚ 4,500+ LOC β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ Core β”‚ β”‚ Search β”‚ β”‚ RAG β”‚ β”‚Infrastructureβ”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (6 files) β”‚ β”‚ (2 files) β”‚ β”‚ (2 files) β”‚ β”‚ (8 files) β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ 2,100+ LOC β”‚ β”‚ 390+ LOC β”‚ β”‚ 891 LOC β”‚ β”‚ 4,300+ LOC β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ Automation β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (5 files) β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ 2,900+ LOC β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ EXTERNAL SERVICES β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚ SearXNG β”‚ β”‚ Redis β”‚ β”‚PostgreSQLβ”‚ β”‚ Cloudflare Workers AI β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚70+ searchβ”‚ β”‚ Caching β”‚ β”‚ Storage β”‚ β”‚ gpt-oss-120b, qwq-32b β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ engines β”‚ β”‚ Sessions β”‚ β”‚ Users β”‚ β”‚ llama, bge-m3, guard β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + Caller + β”‚ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Cloudflare edge (300+ PoPs) β”‚ + β”‚ workers/src/index.ts (Hono) β”‚ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ KV β”‚ β”‚ Workers β”‚ β”‚ Durable β”‚ β”‚ Service β”‚ + β”‚ cache β”‚ β”‚ AI β”‚ β”‚ Objects β”‚ β”‚ binding β†’ β”‚ + β”‚ hit β”‚ β”‚ (LLM) β”‚ β”‚ (Rate- β”‚ β”‚ FastAPI β”‚ + β”‚ β†’ β”‚ β”‚ ↓ β”‚ β”‚ Limiter, β”‚ β”‚ Container β”‚ + β”‚ return β”‚ β”‚ Vector- β”‚ β”‚ Topic- β”‚ β”‚ (apps/ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ ize / D1 β”‚ β”‚ Monitor, β”‚ β”‚ backend/) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Research-β”‚ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ Agent, β”‚ β”‚ + β”‚ Session) β”‚ β–Ό + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ SearXNG (70+ β”‚ + β”‚ engines) β”‚ + β”‚ Postgres origin β”‚ + β”‚ Redis (legacy) β”‚ + β”‚ Stripe webhooks β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–² + β”‚ + Async via Queues + (research fan-out, + monitor checks, + batch crawls) ``` ---- +ASCII diagrams elide a lot. The intended invariants: -## API Endpoints (58 Total) - -### Agent API (Tavily-Compatible) - 5 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/agent/search` | POST | βœ… Working | AI search with model selection | -| `/api/v1/agent/extract` | POST | βœ… Working | Content extraction | -| `/api/v1/agent/research` | POST | βœ… Working | Deep research (exclusive) | -| `/api/v1/agent/models` | GET | βœ… Working | List AI models | -| `/api/v1/agent/health` | GET | βœ… Working | Health check | - -### Search API - 4 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/search/` | GET/POST | βœ… Working | Basic search | -| `/api/v1/search/batch` | POST | βœ… Working | Batch search | -| `/api/v1/search/engines` | GET | βœ… Working | List engines | -| `/api/v1/search/health` | GET | βœ… Working | Health check | - -### RAG API - 8 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/rag/search` | POST | βœ… Working | RAG search | -| `/api/v1/rag/research` | POST | βœ… Working | Research mode | -| `/api/v1/rag/semantic-search` | POST | βœ… Working | Semantic search | -| `/api/v1/rag/corpus` | POST/GET | βœ… Working | Corpus management | -| `/api/v1/rag/corpus/{id}` | DELETE | βœ… Working | Delete corpus | -| `/api/v1/rag/corpus/{id}/info` | GET | βœ… Working | Corpus info | -| `/api/v1/rag/generate-queries` | POST | βœ… Working | Query generation | -| `/api/v1/rag/images` | POST | βœ… Working | Image search | - -### Enhanced API - 7 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/enhanced/scrape` | POST | βœ… Working | Enhanced scraping | -| `/api/v1/enhanced/search` | POST | βœ… Working | Enhanced search | -| `/api/v1/enhanced/chunk-content` | POST | βœ… Working | Content chunking | -| `/api/v1/enhanced/discover-urls` | POST | βœ… Working | URL discovery | -| `/api/v1/enhanced/extract-tables` | POST | βœ… Working | Table extraction | -| `/api/v1/enhanced/features` | GET | βœ… Working | List features | -| `/api/v1/enhanced/performance` | GET | βœ… Working | Performance stats | - -### Advanced v2 API - 14 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/v2/advanced/scrape/advanced` | POST | βœ… Working | Advanced scraping | -| `/api/v1/v2/advanced/scrape/multi-engine` | POST | βœ… Working | Multi-engine scrape | -| `/api/v1/v2/advanced/search/multi-provider` | POST | βœ… Working | Multi-provider search | -| `/api/v1/v2/advanced/extract/attributes` | POST | βœ… Working | Attribute extraction | -| `/api/v1/v2/advanced/extract/multi-entity` | POST | βœ… Working | Entity extraction | -| `/api/v1/v2/advanced/map/website` | POST | ⚠️ Partial | Website mapping | -| `/api/v1/v2/advanced/track/changes` | POST | βœ… Working | Change tracking | -| `/api/v1/v2/advanced/batch/submit` | POST | βœ… Working | Batch operations | -| `/api/v1/v2/advanced/batch/{id}/status` | GET | βœ… Working | Batch status | -| `/api/v1/v2/advanced/batch/{id}/control` | POST | βœ… Working | Batch control | -| `/api/v1/v2/advanced/config/generate` | POST | ⚠️ Needs API Key | LLM config generation | -| `/api/v1/v2/advanced/actions/execute` | POST | βœ… Working | Browser actions | -| `/api/v1/v2/advanced/stats/comprehensive` | GET | βœ… Working | System stats | -| `/api/v1/v2/advanced/health/advanced` | GET | βœ… Working | Advanced health | - -### Auth API - 10 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/auth/register` | POST | βœ… Working | User registration | -| `/api/v1/auth/login` | POST | βœ… Working | Login | -| `/api/v1/auth/refresh` | POST | βœ… Working | Token refresh | -| `/api/v1/auth/me` | GET | βœ… Working | Current user | -| `/api/v1/auth/api-keys` | GET/POST | βœ… Working | API key management | -| `/api/v1/auth/api-keys/{id}` | DELETE | βœ… Working | Delete API key | -| `/api/v1/auth/usage` | GET | βœ… Working | Usage stats | -| `/api/v1/auth/change-password` | POST | βœ… Working | Password change | -| `/api/v1/auth/reset-password` | POST | βœ… Working | Password reset | -| `/api/v1/auth/verify-email` | POST | βœ… Working | Email verification | - -### Billing API - 10 Endpoints βœ… -| Endpoint | Method | Status | Description | -|----------|--------|--------|-------------| -| `/api/v1/billing/plans` | GET | βœ… Working | List plans | -| `/api/v1/billing/subscription` | GET/POST | βœ… Working | Subscription | -| `/api/v1/billing/checkout-session` | POST | βœ… Working | Stripe checkout | -| `/api/v1/billing/billing-portal` | POST | βœ… Working | Billing portal | -| `/api/v1/billing/invoices` | GET | βœ… Working | Invoices | -| `/api/v1/billing/payment-methods` | GET/POST | βœ… Working | Payment methods | -| `/api/v1/billing/webhook/stripe` | POST | βœ… Working | Stripe webhook | +1. **Edge handles what it can.** Auth checks, rate limiting, KV-cache lookups, simple search proxying, and any endpoint with a pure-TypeScript implementation terminate at the worker. No Python round-trip. +2. **Container handles what it must.** Heavy scraping, complex orchestrations (research pipelines, big crawls), Stripe webhook signing, alembic migrations against the long-tail of Postgres-backed features. Everything called from the worker via a [service binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/service-bindings/). +3. **Durable Objects do stateful coordination, never request handling.** A request handler may *create* a DO instance; the DO itself runs to completion in the background and emits events via Queues / webhooks. +4. **Queues are the boundary for "this might take seconds."** Anything that could blow the 50ms Worker CPU budget or the Container's request-timeout gets queued. --- -## Service Layer Architecture - -### 1. AI Services (`app/services/ai/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `cloudflare_ai.py` | 861 | βœ… Complete | Cloudflare Workers AI integration | -| `search_pipeline.py` | 507 | βœ… Complete | End-to-end AI search pipeline | -| `__init__.py` | - | βœ… | Exports | - -**Capabilities:** -- βœ… OpenAI gpt-oss-120b (Responses API) -- βœ… Reasoning models (qwq-32b, deepseek-r1) -- βœ… Quality models (llama-3.3-70b, gemma-3) -- βœ… Speed models (llama-3.1-8b) -- βœ… Embeddings (bge-m3, multilingual) -- βœ… Reranking (bge-reranker) -- βœ… Content safety (llama-guard) -- βœ… Intelligent model selection -- βœ… Chain-of-thought reasoning - -### 2. Scraping Services (`app/services/scraping/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `scraping.py` | 815 | βœ… Complete | Core scraping service | -| `enhanced_scraping.py` | 643 | βœ… Complete | Advanced scraping | -| `multi_engine_scraper.py` | 719 | βœ… Complete | Multi-engine support | -| `playwright_scraping.py` | 539 | βœ… Complete | JavaScript rendering | -| `html_converter.py` | 631 | βœ… Complete | HTML processing | -| `markdown_generation.py` | 665 | βœ… Complete | Markdown output | -| `pdf_processing.py` | 617 | βœ… Complete | PDF extraction | -| `puppeteer_client.py` | 33 | ⚠️ Stub | Puppeteer integration | - -**Capabilities:** -- βœ… Static HTML scraping (BeautifulSoup) -- βœ… JavaScript rendering (Playwright) -- βœ… PDF extraction -- βœ… Markdown conversion -- βœ… Multi-engine parallel scraping -- βœ… Robots.txt compliance -- βœ… User-agent rotation - -### 3. Extraction Services (`app/services/extraction/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `extraction_strategies.py` | 700+ | βœ… Complete | Extraction strategies | -| `chunking_strategies.py` | 600+ | βœ… Complete | Content chunking | -| `ai_extraction.py` | 500+ | βœ… Complete | AI-powered extraction | -| `attributes_extraction.py` | 500+ | βœ… Complete | Attribute extraction | -| `table_extraction.py` | 500+ | βœ… Complete | Table extraction | -| `multi_entity_extraction.py` | 500+ | βœ… Complete | Entity extraction | -| `content_filters.py` | 400+ | βœ… Complete | Content filtering | -| `link_analysis.py` | 400+ | βœ… Complete | Link analysis | - -**Capabilities:** -- βœ… LLM-based extraction -- βœ… CSS/XPath selectors -- βœ… Schema-based extraction -- βœ… Table extraction (HTML tables) -- βœ… Multi-entity extraction -- βœ… Content chunking strategies -- βœ… Boilerplate removal - -### 4. Crawling Services (`app/services/crawling/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `deep_crawling.py` | 700+ | βœ… Complete | Deep crawling | -| `website_mapping.py` | 600+ | βœ… Complete | Site mapping | -| `adaptive_crawling.py` | 600+ | βœ… Complete | Adaptive crawling | -| `change_tracking.py` | 500+ | βœ… Complete | Change detection | -| `crawl_management.py` | 500+ | βœ… Complete | Crawl management | -| `url_seeder.py` | 400+ | βœ… Complete | URL seeding | -| `virtual_scrolling.py` | 400+ | βœ… Complete | Infinite scroll | -| `crawler_monitor.py` | 300+ | βœ… Complete | Monitoring | - -**Capabilities:** -- βœ… Deep crawling -- βœ… Website mapping -- βœ… Change tracking -- βœ… Adaptive rate limiting -- βœ… Virtual scrolling (infinite scroll pages) -- βœ… Crawl scheduling - -### 5. Infrastructure Services (`app/services/infrastructure/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `batch_operations.py` | 700+ | βœ… Complete | Batch processing | -| `dispatcher.py` | 664 | βœ… Complete | Task dispatching | -| `link_preview.py` | 672 | βœ… Complete | Link previews | -| `proxy_rotation.py` | 534 | βœ… Complete | Proxy rotation | -| `user_agent_generator.py` | 639 | βœ… Complete | UA generation | -| `webhook_integration.py` | 659 | βœ… Complete | Webhooks | -| `zero_retention.py` | 599 | βœ… Complete | Privacy mode | - -**Capabilities:** -- βœ… Batch job processing -- βœ… Proxy rotation -- βœ… User-agent rotation -- βœ… Webhook notifications -- βœ… Zero-retention mode - -### 6. Automation Services (`app/services/automation/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `actions_system.py` | 800+ | βœ… Complete | Browser actions | -| `llm_configuration.py` | 700+ | ⚠️ Needs API Key | LLM config | -| `browser_config.py` | 500+ | βœ… Complete | Browser config | -| `browser_profiler.py` | 400+ | βœ… Complete | Browser profiling | - -**Capabilities:** -- βœ… Click, type, scroll actions -- βœ… Form filling -- βœ… Screenshot capture -- βœ… JavaScript execution -- ⚠️ LLM-powered configuration (needs API key) - -### 7. Core Services (`app/services/core/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `searxng.py` | 600+ | βœ… Complete | SearXNG integration | -| `database.py` | 500+ | βœ… Complete | Database access | -| `cache.py` | 500+ | βœ… Complete | Redis caching | -| `database_manager.py` | 300+ | βœ… Complete | DB management | -| `cache_context.py` | 200+ | βœ… Complete | Cache context | - -### 8. RAG Services (`app/services/rag/`) -| File | Lines | Status | Purpose | -|------|-------|--------|---------| -| `rag.py` | 871 | βœ… Complete | RAG pipeline | - -**Capabilities:** -- βœ… Embedding generation (Cloudflare AI / OpenAI) -- βœ… Vector store (in-memory) -- βœ… Semantic search -- βœ… Research mode -- βœ… Query generation +## Components ---- +### Edge worker β€” `workers/` + +- **Stack:** TypeScript, Hono router on Cloudflare Workers. +- **Files of interest:** `workers/src/index.ts` (router), `workers/src/routes/*.ts` (per-feature handlers), `workers/src/durable-objects/*.ts`, `workers/src/queue-consumer.ts`, `workers/src/scheduled.ts`, `workers/src/middleware/*.ts`, `workers/wrangler.toml` (bindings), `workers/schema.sql` (D1 schema), `workers/containers.toml` (Container config). +- **Bindings:** `DB` (D1), `CACHE` (KV β€” auth/ratelimit/search cache), `BUCKET` (R2), `VECTORS` (Vectorize), `AI` (Workers AI), `QUEUE` (Cloudflare Queues), plus DO namespaces `RATE_LIMITER`, `TOPIC_MONITOR`, `RESEARCH_AGENT`, `SESSION_MANAGER`, and a service binding `BACKEND` β†’ FastAPI Container. +- **Per-route file map:** `agent.ts` (Tavily-compat), `auth.ts`, `billing.ts`, `knowledge.ts`, `monitor.ts`, `neural.ts`, `proxy.ts` (catch-all β†’ Container), `rag.ts`, `search.ts`, `verify.ts`. + +### Backend Container β€” `app/` and `apps/backend/` + +- **Stack:** Python 3.11+, FastAPI, Uvicorn, Pydantic v2, SQLAlchemy + Alembic (Postgres), Celery (Redis), httpx for outbound calls. 93 endpoints across 14 routers (counted via `app/api/v1/*.py` and `app/api/v2/*.py`). +- **Two paths, same code.** `app/` is the historic single-package layout still imported by production (`uvicorn app.main:app`). `apps/backend/` is the monorepo-shaped mirror that builds the Docker image and ships under [`Dockerfile.cloudflare`](../Dockerfile.cloudflare). A future PR will collapse them β€” see [ADR-0006](./adr/0006-monorepo-with-apps-and-workers.md). +- **Service layout:** `app/services/core/` (DB, KV, queues, D1 client), `app/services/search/` (SearXNG orchestration + dedup + rerank), `app/services/scraping/` (static + JS + PDF), `app/services/extraction/` (entities, tables, attributes), `app/services/crawling/`, `app/services/rag/`, `app/services/ai/` (Workers AI client + model-tier selector). +- **Inbound interface:** invoked by the edge worker via service binding for endpoints that aren't pure-TypeScript. The Container does *not* face the public internet directly in production. + +### Web dashboard β€” `apps/web/` + +- **Stack:** Next.js 15 (App Router) on Cloudflare Workers via `@opennextjs/cloudflare`. Tailwind for styling. Routes under `app/(auth)/` (login, signup), `app/(dashboard)/` (dashboard, api-keys, playground, billing). +- **Deploy:** Cloudflare Workers via `pnpm cf:build && pnpm cf:deploy` (config in `apps/web/wrangler.toml`). The migration from Cloudflare Pages to native Workers landed in commit [`376f886`](https://github.com/Rakesh1002/unsearch/commit/376f886) and is documented in `CHANGELOG.md`. +- **API client:** uses `@unsearch/sdk` directly β€” no separate fetch layer. + +### SDKs β€” `apps/sdk-*/` + +| Package | Languages | Purpose | +|---------|-----------|---------| +| [`@unsearch/sdk`](../apps/sdk-ts/) | TypeScript / Node / Bun / Deno / Workers / Edge | Primary public SDK, mirrors REST surface 1:1 | +| [`unsearch`](../apps/sdk-py/) | Python 3.9–3.13 (sync + async) | See [ADR-0007](./adr/0007-python-sdk-sync-and-async.md) | +| [`@unsearch/llamaindex`](../apps/sdk-llamaindex/) | TypeScript | LlamaIndex `BaseRetriever` implementation backed by UnSearch | + +All three are kept structurally parallel (same method names, same request/response shapes, same SSE-streaming pattern) so cross-language onboarding is mechanical. -## Infrastructure +### SearXNG β€” `searxng/` -### Docker Services -| Service | Purpose | Status | -|---------|---------|--------| -| `api` | FastAPI application | βœ… Running | -| `searxng` | Meta-search (70+ engines) | βœ… Running | -| `redis` | Caching, sessions | βœ… Running | -| `postgres` | User data, API keys | βœ… Running | -| `flower` | Celery monitoring | ⚠️ Optional | -| `nginx` | Reverse proxy | ⚠️ Optional | +The 70+-engine meta-search aggregator. Self-hosted as a Docker container (`docker-compose.yml`) in production and in self-host deployments. The FastAPI backend in `app/services/search/` is the only client. Configuration in `searxng/settings.yml`. See [ADR-0002](./adr/0002-searxng-as-meta-search-aggregator.md). -### External Services -| Service | Purpose | Status | -|---------|---------|--------| -| Cloudflare Workers AI | LLM, embeddings, safety | βœ… Configured | -| Stripe | Billing (optional) | ⚠️ Needs config | +### Origin Postgres + Redis + +Postgres is the Container's origin database for everything that hasn't moved to D1 yet β€” primarily the legacy Stripe/billing state and the Celery task results table. Redis backs Celery's broker and Celery's result backend. Both run as containers in self-host (`docker-compose.yml`) and as managed services (Neon Postgres, Upstash Redis) in production. + +### Monitoring β€” `monitoring/` + +Prometheus + Grafana, provisioned via `monitoring/docker-compose.monitoring.yml`. Dashboards in `monitoring/grafana/dashboards/unsearch-overview.json`. The Container exports Prometheus metrics at `/metrics`; the worker emits the same via Workers Analytics Engine. Detailed observability runbook: [`workers/OBSERVABILITY.md`](../workers/OBSERVABILITY.md). --- -## Code Statistics +## Data model β€” what lives where -| Category | Files | Lines of Code | -|----------|-------|---------------| -| Services | 50+ | 27,500+ | -| API | 10+ | 3,000+ | -| Models | 6 | 1,500+ | -| Utils | 6 | 2,000+ | -| Config | 1 | 300+ | -| **Total** | **70+** | **34,000+** | +| State | Store | Why | +|-------|-------|-----| +| Users, accounts, plans | D1 (`workers/schema.sql`) β€” primary; Postgres mirror for the long-tail of Container-only features | Most reads happen at the edge; users + plans are the hot path | +| API keys | D1 | Auth check is on every request; must be edge-fast | +| Stripe subscriptions, invoices | Postgres (Container) β€” D1 stores only the `customer_id` / `subscription_id` projection | Stripe webhooks land at the Container; full state is too rich for D1 | +| Rate-limit counters | KV (TTL) + DO `RateLimiter` for sliding-window | KV for absolute-limit checks; DO for per-key sliding window | +| Search result cache | KV (60s default, configurable per-plan) | Reads are cache-hit-or-recompute; per-request size fits KV's value-size limit | +| Embeddings + metadata | Vectorize (`@cf/baai/bge-m3`, 1024d) | First-party vector store with edge-bindings | +| Scraped HTML + PDFs | R2 | Variable size, long retention, infrequent reads | +| Async job state | DO `ResearchAgent`, `TopicMonitor` (alarms), Queue messages | Each long-running operation owns its own DO; queue for dispatch | +| Chat / pagination cursors | DO `SessionManager` | Per-user state with TTL eviction | + +The Container reaches D1 / KV / Vectorize / Queues over **REST**, not via direct bindings, because Cloudflare Containers do not yet expose direct bindings from inside the Container runtime. The REST clients live in `app/services/core/d1_client.py`, `cache_kv.py`, and `queue_producer.py`. The worker uses direct bindings β€” no REST hop. --- -## Integration Status +## Request lifecycle examples + +### Cache hit on `POST /api/v1/search` -### Fully Integrated βœ… -- [x] Cloudflare Workers AI (all models) -- [x] SearXNG (70+ search engines) -- [x] Redis caching -- [x] PostgreSQL storage -- [x] Authentication system -- [x] API key management -- [x] Rate limiting +1. Worker `index.ts` matches the route, calls `routes/search.ts`. +2. KV lookup with key `search:sha256(query+engines+filters)`. +3. **Hit** β†’ return the cached `SearchResponse` with `cache_hit: true`. ~5ms p95. Worker CPU budget unused. -### Partially Integrated ⚠️ -- [ ] Stripe billing (needs API key) -- [ ] LLM config generation (needs OpenAI key) -- [ ] Puppeteer (stub only) -- [ ] Website mapping (validation issues) +### Cache miss on `POST /api/v1/search` -### Not Integrated ❌ -- [ ] External vector database (using in-memory) -- [ ] Celery workers (disabled) -- [ ] Email notifications +1. Worker β†’ SearXNG via the Container service binding. +2. Container `app/api/v1/search.py` orchestrates `app/services/search/`: parallel engine queries, dedup, optional rerank via Workers AI (worker-resident β€” Container calls back to the worker for AI inference). +3. Optional scraping if `scrape_content: true` (`app/services/scraping/`). +4. Container writes the response back to KV via REST (`cache_kv.py`). +5. Container returns response to the worker, worker returns to caller. + +### `POST /api/v1/agent/research` + +1. Worker `routes/agent.ts` validates auth + plan. +2. Worker spawns / fetches a `ResearchAgent` Durable Object instance keyed by `session_id`. +3. DO returns `{session_id, status: "running"}` immediately to the caller. +4. DO loops: query expansion β†’ SearXNG search β†’ scrape candidates β†’ Workers AI synthesis β†’ repeat until depth limit or convergence. Each step writes its results back to the DO's internal SQLite. +5. Caller polls `GET /api/v1/agent/research/{id}` β†’ worker reads DO state β†’ returns. When `status: "completed"`, the `finalAnswer` field is populated. + +### Topic-monitor webhook fire + +1. Caller creates monitor via `POST /api/v1/monitor/topics` β†’ worker spawns a `TopicMonitor` DO with `interval_minutes` and `webhook_url`. +2. DO sets an alarm. On wake, runs the monitored search, computes a delta against the last result, and **enqueues** the webhook delivery via Cloudflare Queues. +3. The `queue-consumer.ts` worker reads the queue, POSTs to the webhook with retry + exponential backoff, and writes the delivery receipt back to the DO. --- -## API Compatibility +## Performance targets -| Platform | Compatibility | Notes | -|----------|---------------|-------| -| Tavily | βœ… 100% | Drop-in replacement | -| LangChain | βœ… Full | SDK provided | -| LlamaIndex | ⚠️ Planned | Not yet implemented | -| OpenAI | ⚠️ Partial | Similar format | +| Metric | Target | Current | +|--------|--------|---------| +| API endpoints | 93 across 14 routers | βœ… | +| p95 search latency (KV cache hit) | <100ms | ~80ms | +| p95 search latency (miss) | <2s | 1–3s | +| AI answer generation | tier-dependent | `fast`: 1–3s, `balanced`: 3–8s, `reasoning`: 8–20s | +| Scraping throughput | 10 URLs/s per Container replica | βœ… | +| Container cold start | <2s | ~1.5s | +| Test coverage | >80% | ~40% (tech debt) | +| Uptime | 99.9% | tracked via [`workers/OBSERVABILITY.md`](../workers/OBSERVABILITY.md) | --- -## Security Features +## Tech-debt callouts + +From [docs/roadmap.md Β§ Technical Debt](./roadmap.md): -| Feature | Status | -|---------|--------| -| API key authentication | βœ… | -| JWT tokens | βœ… | -| Rate limiting | βœ… | -| Zero-retention mode | βœ… | -| Content safety checks | βœ… | -| CORS configuration | βœ… | -| Input validation | βœ… | +| Issue | Location | Impact | Plan | +|-------|----------|--------|------| +| In-memory vector store | `app/services/rag/rag.py` | Doesn't scale beyond a small corpus | Migrate fully to Vectorize (most paths already use it; a few are still legacy) | +| Puppeteer stub | `app/services/scraping/puppeteer_client.py` | No fallback when CF Browser Rendering unavailable | Either wire CF Browser Rendering or drop the stub | +| Sync DB operations | Multiple Container files | Blocks Uvicorn worker threads under load | Move to async SQLAlchemy where it matters | +| Two backend layouts (`app/` vs `apps/backend/`) | Repo root vs monorepo | Onboarding confusion | Collapse β€” separate PR β€” see [ADR-0006](./adr/0006-monorepo-with-apps-and-workers.md) | --- -## Performance +## What this doc doesn't cover -| Metric | Value | -|--------|-------| -| API endpoints | 58 | -| Concurrent requests | 100+ | -| Search latency (cached) | <100ms | -| Search latency (uncached) | 1-3s | -| AI answer generation | 2-35s (model dependent) | -| Scraping throughput | 10 URLs/s | +- **Per-endpoint contracts.** See [`API_REFERENCE.md`](./API_REFERENCE.md) and the live OpenAPI at `/docs`. +- **Per-decision rationale.** See [`adr/`](./adr/README.md). +- **Honest feature status.** See [`feature-matrix.md`](./feature-matrix.md) and [ADR-0008](./adr/0008-honest-feature-status-policy.md). +- **Deploy step-by-step.** See [`deployment/`](./deployment/) and [`quickstart.md`](./quickstart.md). +- **On-call playbook.** See [`operations/RUNBOOKS.md`](./operations/RUNBOOKS.md). +- **Where does that file live?** See [`what-is-what.md`](./what-is-what.md). diff --git a/docs/configuration/env-variables.md b/docs/configuration/env-variables.md index 74c959c..734065c 100644 --- a/docs/configuration/env-variables.md +++ b/docs/configuration/env-variables.md @@ -40,7 +40,7 @@ STRIPE_PRO_PRICE_ID=price_... # Create in Stripe Dashboard # SearXNG Configuration SEARXNG_URL=http://localhost:8080 # Default if using Docker -SEARXNG_SECRET=ultrasecretkey # Must match searxng/settings.yml +SEARXNG_SECRET=change-me-with-openssl-rand-hex-32 # Must match searxng/settings.yml # CORS Configuration ALLOWED_ORIGINS=["http://localhost:3000","https://app.unsearch.dev"] @@ -148,7 +148,7 @@ services: searxng: environment: - - SEARXNG_SECRET=ultrasecretkey + - SEARXNG_SECRET=change-me-with-openssl-rand-hex-32 - SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml ``` diff --git a/docs/what-is-what.md b/docs/what-is-what.md new file mode 100644 index 0000000..04523fb --- /dev/null +++ b/docs/what-is-what.md @@ -0,0 +1,172 @@ +# What is what + +> Plain-language map of the repo so a new contributor can navigate in 5 minutes. For the *why*, see [`adr/`](./adr/README.md). For the runtime picture, see [`architecture.md`](./architecture.md). + +## Top-level tour + +``` +unsearch/ +β”œβ”€β”€ README.md ← Start here. Repo overview + quick start. +β”œβ”€β”€ CHANGELOG.md ← What shipped in each release. Source of truth for release notes. +β”œβ”€β”€ CLAUDE.md ← Per-repo conventions for Claude Code sessions (and human contributors). +β”œβ”€β”€ CONTRIBUTING.md ← How to propose changes. PR template, test expectations. +β”œβ”€β”€ LICENSE ← Apache 2.0. See ADR-0005. +β”‚ +β”œβ”€β”€ app/ ← Python FastAPI backend (legacy single-package layout). uvicorn imports from here. +β”œβ”€β”€ apps/ ← Monorepo packages. +β”‚ β”œβ”€β”€ backend/ ← FastAPI backend, monorepo-shaped. Builds the Docker image. +β”‚ β”œβ”€β”€ web/ ← Next.js dashboard on Cloudflare Workers (@opennextjs/cloudflare). +β”‚ β”œβ”€β”€ sdk-ts/ ← @unsearch/sdk β€” TypeScript SDK. +β”‚ β”œβ”€β”€ sdk-py/ ← unsearch β€” Python SDK (sync + async). See ADR-0007. +β”‚ └── sdk-llamaindex/ ← @unsearch/llamaindex β€” LlamaIndex retriever. +β”‚ +β”œβ”€β”€ workers/ ← Cloudflare Workers edge router (Hono) + Durable Objects + D1 schema. +β”‚ +β”œβ”€β”€ docs/ ← All long-form docs. +β”‚ β”œβ”€β”€ README.md ← This directory's index. +β”‚ β”œβ”€β”€ architecture.md ← Current architecture overview. +β”‚ β”œβ”€β”€ what-is-what.md ← ← you are here. +β”‚ β”œβ”€β”€ adr/ ← Architecture Decision Records. +β”‚ β”œβ”€β”€ feature-matrix.md ← Honest βœ… / πŸ”Ά / πŸ“‹ status per feature. Source of truth for marketing claims. +β”‚ β”œβ”€β”€ roadmap.md ← ICP-ordered priorities. P0 β†’ P4. +β”‚ β”œβ”€β”€ strategy/ ← ICP, GTM, pricing, positioning, user journey. +β”‚ β”œβ”€β”€ operations/ ← Runbooks. Read before on-call. +β”‚ β”œβ”€β”€ deployment/ ← Per-target deploy guides (Cloudflare, Railway, DigitalOcean). +β”‚ β”œβ”€β”€ configuration/ ← Env vars + Stripe webhook setup. +β”‚ β”œβ”€β”€ migration/ ← Migration guides from Tavily/Exa. +β”‚ β”œβ”€β”€ API_REFERENCE.md ← Full endpoint catalog. Live OpenAPI at /docs. +β”‚ β”œβ”€β”€ API_EXAMPLES.md ← Worked examples per endpoint. +β”‚ β”œβ”€β”€ ai-pipeline.md ← Models, embeddings, reranking. See ADR-0004. +β”‚ β”œβ”€β”€ cloudflare-architecture.md ← Edge / Containers / D1 / Vectorize wiring detail. See ADR-0001. +β”‚ β”œβ”€β”€ BILLING_SETUP.md ← Stripe products + prices + portal config. +β”‚ β”œβ”€β”€ SECRETS_MANAGEMENT.md ← How we handle env secrets in self-host and prod. +β”‚ β”œβ”€β”€ introduction.mdx ← Public docs site entry (Mintlify). +β”‚ └── quickstart.md ← Self-host quickstart. +β”‚ +β”œβ”€β”€ alembic/ ← Postgres migrations (origin DB). +β”œβ”€β”€ searxng/ ← SearXNG meta-search engine config. See ADR-0002. +β”œβ”€β”€ monitoring/ ← Prometheus + Grafana provisioning. Dashboards in grafana/dashboards/. +β”œβ”€β”€ nginx/ ← Reverse-proxy config for self-host TLS. +β”œβ”€β”€ scripts/ ← Setup + ops scripts (manage.sh, setup-stripe.sh, smoke tests). +β”œβ”€β”€ tests/ ← Backend test suite (unit + integration + performance). +β”‚ +β”œβ”€β”€ docker-compose.yml ← Self-host: full stack (API + SearXNG + Postgres + Redis + Celery). +β”œβ”€β”€ docker-compose.quickstart.yml ← Self-host: minimal stack (API + SearXNG only). +β”œβ”€β”€ docker-compose.prod.yml ← Self-host: production overrides. +β”œβ”€β”€ Dockerfile ← FastAPI Container image (self-host). +β”œβ”€β”€ Dockerfile.cloudflare ← FastAPI Container image (Cloudflare Containers). +β”‚ +β”œβ”€β”€ .env.example ← Copy to .env. Required: CLOUDFLARE_ACCOUNT_ID + CLOUDFLARE_API_TOKEN. +└── requirements.txt ← Backend Python deps. apps/sdk-py has its own pyproject.toml. +``` + +## Layer by layer β€” what runs what + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + Public β”‚ apps/web (Next.js β†’ Workers)β”‚ app.unsearch.dev + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + Edge β”‚ workers/ (Hono on Workers) β”‚ api.unsearch.dev + β”‚ + Durable Objects (RateLimiter,β”‚ + β”‚ TopicMonitor, ResearchAgent, β”‚ + β”‚ SessionManager) β”‚ + β”‚ + Queues consumer β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ service binding + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + Container β”‚ app/ + apps/backend/ β”‚ FastAPI on CF Containers + β”‚ (Python 3.11, 93 endpoints) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + State β”‚ Cloudflare β”‚ Origin β”‚ Postgres β”‚ + β”‚ D1 / KV / β”‚ β”‚ + Redis β”‚ + β”‚ R2 / Vec β”‚ β”‚ + SearXNG β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Glossary + +Terms that show up in code and docs without being defined. + +| Term | What it means here | +|------|--------------------| +| **Edge worker** | The Cloudflare Workers script in `workers/`. Fronts every request. | +| **Container** | The Cloudflare Containers FastAPI deployment. Same code as `app/`, packaged via `Dockerfile.cloudflare`. | +| **DO** | Durable Object. Cloudflare's per-instance stateful actor. We have four β€” see `workers/src/durable-objects/`. | +| **KV** | Cloudflare's eventually-consistent key/value store. Used for hot caches (auth, ratelimit, search). | +| **D1** | Cloudflare's edge SQL database (SQLite dialect). Primary store for users, plans, API keys. | +| **R2** | Cloudflare's object storage (S3-compatible). Stores scraped HTML and PDFs. | +| **Vectorize** | Cloudflare's vector database. Stores `bge-m3` embeddings, 1024 dims. | +| **Queues** | Cloudflare's managed message queue. Used for monitor-fire fan-out and async research steps. | +| **Service binding** | A Workers-to-Workers (or Worker-to-Container) call that skips the public internet. Worker β†’ Container in our case. | +| **Tier (model_tier)** | The four Workers AI tiers: `fast`, `balanced`, `reasoning`, `production`. See ADR-0004. | +| **Drop-in (Tavily-compatible)** | Endpoints under `/api/v1/agent/*` that mirror Tavily's request/response shape 1:1. See ADR-0003. | +| **Namespace (in `/api/v1/rag/*`)** | A logical partition inside Vectorize. Each customer's RAG corpus lives in their own namespace. | +| **`uns_` API key** | Customer-facing API key. Header: `X-API-Key: uns_...`. | +| **πŸ”Ά in beta** | Feature exists in code but is being hardened. See ADR-0008. | +| **πŸš€ differentiator** | Feature that no closed-source competitor has. Used in `docs/feature-matrix.md`. | +| **Persona A / B / C** | The three ICPs. A = indie dev (Maya). B = Seed/A CTO (Priya). C = enterprise buyer (David). Defined in [`strategy/icp.md`](./strategy/icp.md). | + +## "Where does that live?" cheat sheet + +| You want to… | Open this | +|--------------|-----------| +| Read an endpoint's contract | [`API_REFERENCE.md`](./API_REFERENCE.md) or `http://localhost:8000/docs` | +| See an endpoint's worked example | [`API_EXAMPLES.md`](./API_EXAMPLES.md) | +| Change request validation | `app/models/requests.py` | +| Change a route handler at the edge | `workers/src/routes/*.ts` | +| Change a route handler in the Container | `app/api/v1/*.py` | +| Add a Durable Object | `workers/src/durable-objects/` + binding in `workers/wrangler.toml` | +| Add a new Cloudflare resource (KV, R2, etc.) | `workers/wrangler.toml` + a Python REST client in `app/services/core/` | +| Tweak Workers AI model selection | `app/services/ai/` (Container) + ADR-0004 | +| Touch the SearXNG engine config | `searxng/settings.yml` | +| Touch DB schema (edge) | `workers/schema.sql` (D1) | +| Touch DB schema (origin) | `alembic/versions/*.py` then `alembic upgrade head` | +| Update the dashboard | `apps/web/app/` | +| Update the public SDK contracts | All three of: `apps/sdk-ts/src/index.ts`, `apps/sdk-py/src/unsearch/client.py`, `apps/sdk-llamaindex/src/index.ts` | +| Add a Stripe product | `scripts/setup-stripe.sh` then run it | +| Add a Stripe webhook | [`configuration/stripe-webhook.md`](./configuration/stripe-webhook.md) | +| Wire a new env var | `.env.example` (root) + `docs/configuration/env-variables.md` + the consuming code | +| Trigger a deploy | [`deployment/quick-reference.md`](./deployment/quick-reference.md) | +| Investigate a prod incident | [`operations/RUNBOOKS.md`](./operations/RUNBOOKS.md) | +| Add a non-obvious architectural decision | [`adr/README.md`](./adr/README.md) | + +## Two backend layouts β€” why? + +`app/` and `apps/backend/` contain the same code. This isn't a bug; it's an in-flight migration. See [ADR-0006](./adr/0006-monorepo-with-apps-and-workers.md) for the explanation. Until they're collapsed: + +- **`app/`** is what `uvicorn app.main:app` imports. Production currently runs from here. +- **`apps/backend/`** is what `Dockerfile.cloudflare` packages. Container deploys use this path. +- **Don't pick favorites.** If you change one, check both. A future PR will collapse them. + +## Naming conventions + +| Pattern | Where | Example | +|---------|-------|---------| +| `snake_case` | Python, JSON request/response fields | `model_tier`, `cache_hit` | +| `camelCase` | TypeScript SDK methods, internal | `neuralSearch`, `tavilySearch` | +| `kebab-case` | URL paths, npm packages, docker images | `/api/v1/agent/research`, `@unsearch/sdk` | +| `PascalCase` | TypeScript classes, Python classes | `UnSearch`, `AsyncUnSearch`, `RateLimiter` | +| `SCREAMING_SNAKE` | Env vars | `CLOUDFLARE_API_TOKEN` | + +When the same field crosses a language boundary (e.g., a Python TypedDict that mirrors a TS interface), we always pick **snake_case for the wire format** and let each language convert as idiomatic. + +## How to read a PR + +When a PR lands, the order of relevant files is usually: + +1. `app/api/v1/*.py` or `workers/src/routes/*.ts` β€” the route handler +2. `app/services/*` β€” supporting service code +3. `apps/sdk-*/` β€” if the wire format changed, the SDKs follow in lockstep +4. `docs/API_REFERENCE.md` + `docs/feature-matrix.md` β€” if the docs are out of date, request changes +5. `CHANGELOG.md` β€” `[Unreleased]` section should reflect the change +6. `docs/adr/` β€” only for cross-cutting decisions, not feature work + +If a PR adds a new architectural decision and *doesn't* include an ADR, that's a review comment ("please write an ADR-NNNN for this"). diff --git a/stripe_webhook_setup.md b/stripe_webhook_setup.md deleted file mode 100644 index ad783c0..0000000 --- a/stripe_webhook_setup.md +++ /dev/null @@ -1,85 +0,0 @@ -# πŸͺ Stripe Webhook Setup Guide - -## Method 1: Manual Setup (Stripe Dashboard) - -### Step 1: Create Webhook Endpoint -1. Go to [Stripe Dashboard > Webhooks](https://dashboard.stripe.com/test/webhooks) -2. Click "+ Add endpoint" -3. Enter your endpoint URL: `https://your-domain.com/api/v1/billing/webhook/stripe` - -### Step 2: Select Events -Add these events (copy-paste friendly): -``` -customer.subscription.created -customer.subscription.updated -customer.subscription.deleted -invoice.paid -invoice.payment_failed -payment_intent.succeeded -payment_method.attached -checkout.session.completed -``` - -### Step 3: Get Webhook Secret -1. Click on your created webhook -2. Click "Reveal" under "Signing secret" -3. Copy the webhook secret (starts with `whsec_`) -4. Add to your .env file: - ```bash - STRIPE_WEBHOOK_SECRET="whsec_your_secret_here" - ``` - -## Method 2: Automated Setup (CLI) - -### Prerequisites -```bash -# Install Stripe CLI -brew install stripe/stripe-cli/stripe -# OR download from: https://github.com/stripe/stripe-cli/releases -``` - -### Run Setup Script -```bash -./scripts/setup-stripe.sh -``` - -## Method 3: Local Development - -### For testing locally with ngrok: -```bash -# Install ngrok -brew install ngrok -# OR download from: https://ngrok.com/download - -# Expose local server -ngrok http 8000 - -# Use the ngrok URL in webhook setup: -# https://abc123.ngrok.io/api/v1/billing/webhook/stripe -``` - -### For testing with Stripe CLI: -```bash -# Forward webhooks to local endpoint -stripe listen --forward-to localhost:8000/api/v1/billing/webhook/stripe - -# This will show webhook events in real-time -``` - -## Testing Your Webhook - -### Test Events -```bash -# Test subscription created -stripe trigger customer.subscription.created - -# Test payment succeeded -stripe trigger payment_intent.succeeded - -# Test invoice paid -stripe trigger invoice.payment_succeeded -``` - -### Monitor Webhook Logs -- Dashboard: https://dashboard.stripe.com/test/webhooks -- Or check your application logs for webhook processing diff --git a/webhook_events_explained.md b/webhook_events_explained.md deleted file mode 100644 index 66986e5..0000000 --- a/webhook_events_explained.md +++ /dev/null @@ -1,58 +0,0 @@ -# πŸ“‹ Webhook Events Explained - -## Why Each Event is Important for UnSearch API - -### πŸ”„ **Subscription Lifecycle Events** - -| Event | Purpose | What it does in your app | -|-------|---------|-------------------------| -| `customer.subscription.created` | New subscription | Creates subscription record, activates user plan | -| `customer.subscription.updated` | Plan changes, renewals | Updates limits, plan details, billing cycle | -| `customer.subscription.deleted` | Cancellations | Deactivates subscription, reverts to free plan | - -### πŸ’° **Payment Events** - -| Event | Purpose | What it does in your app | -|-------|---------|-------------------------| -| `invoice.paid` | Successful payment | Confirms payment, extends subscription period | -| `invoice.payment_failed` | Failed payment | Handles dunning, may suspend account | -| `payment_intent.succeeded` | One-time payments | Processes credits, upgrades, etc. | - -### 🎯 **Customer Events** - -| Event | Purpose | What it does in your app | -|-------|---------|-------------------------| -| `payment_method.attached` | New card added | Updates default payment method | -| `checkout.session.completed` | Purchase completed | Links payment to user account | - -## πŸ›‘οΈ Security Features - -Your webhook handler includes: - -βœ… **Signature Verification**: Validates requests come from Stripe -βœ… **Idempotency**: Prevents duplicate event processing -βœ… **Event Logging**: Tracks all webhook events for debugging -βœ… **Error Handling**: Graceful failure handling - -## πŸ” Monitoring Webhook Health - -Check webhook status: -- **Dashboard**: https://dashboard.stripe.com/test/webhooks -- **Logs**: Your app logs webhook processing -- **CLI**: `stripe events list` to see recent events - -## ⚑ Testing Tips - -```bash -# Test all critical events -stripe trigger customer.subscription.created -stripe trigger customer.subscription.updated -stripe trigger invoice.payment_succeeded -stripe trigger invoice.payment_failed -``` - -Your app will automatically: -- Create/update user subscriptions -- Adjust API limits and features -- Handle payment failures gracefully -- Maintain accurate billing records