refactor: cluster backend under backend/, ops under infra/

.github/workflows/ci-cd.yml

@@ -35,20 +35,20 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r backend/requirements.txt
           pip install black flake8 isort mypy
 
       - name: Run Black formatter check
-        run: black --check app/ tests/
+        run: black --check backend/app backend/tests
 
       - name: Run isort check
-        run: isort --check-only app/ tests/
+        run: isort --check-only backend/app backend/tests
 
       - name: Run Flake8 linter
-        run: flake8 app/ tests/ --max-line-length=120 --exclude=venv,__pycache__
+        run: flake8 backend/app backend/tests --max-line-length=120 --exclude=venv,__pycache__
 
       - name: Run MyPy type checker
-        run: mypy app/ --ignore-missing-imports
+        run: mypy backend/app --ignore-missing-imports
 
   # ========================================
   # Security Scanning
@@ -76,7 +76,7 @@ jobs:
       - name: Check for security issues with Bandit
         run: |
           pip install bandit
-          bandit -r app/ -f json -o bandit-report.json || true
+          bandit -r backend/app -f json -o bandit-report.json || true
 
       - name: Upload Bandit results
         uses: actions/upload-artifact@v4
@@ -128,7 +128,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r backend/requirements.txt
           pip install pytest-cov
 
       - name: Run unit tests with coverage
@@ -137,7 +137,7 @@ jobs:
           REDIS_URL: redis://localhost:6379
           TEST_MODE: "true"
         run: |
-          pytest tests/unit/ -v --cov=app --cov-report=xml --cov-report=html
+          pytest backend/tests/unit/ -v --cov=backend/app --cov-report=xml --cov-report=html
 
       - name: Upload coverage reports
         uses: codecov/codecov-action@v4
@@ -209,7 +209,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r backend/requirements.txt
 
       - name: Run integration tests
         env:
@@ -218,7 +218,7 @@ jobs:
           SEARXNG_URL: http://localhost:8080
           TEST_MODE: "true"
         run: |
-          pytest tests/integration/ -v --maxfail=5
+          pytest backend/tests/integration/ -v --maxfail=5
 
   # ========================================
   # Performance Tests
@@ -240,7 +240,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r backend/requirements.txt
           pip install locust
 
       - name: Start services with Docker Compose
@@ -250,11 +250,11 @@ jobs:
 
       - name: Run performance tests
         run: |
-          pytest tests/performance/ -v --benchmark-only
+          pytest backend/tests/performance/ -v --benchmark-only
 
       - name: Run load tests with Locust
         run: |
-          locust -f tests/performance/locustfile.py --headless --users 100 --spawn-rate 10 --run-time 60s --host http://localhost:8000
+          locust -f backend/tests/performance/locustfile.py --headless --users 100 --spawn-rate 10 --run-time 60s --host http://localhost:8000
 
       - name: Stop services
         run: docker compose down
@@ -343,7 +343,7 @@ jobs:
       - name: Run smoke tests
         run: |
           pip install pytest httpx
-          pytest tests/smoke/ -v --base-url=https://staging-api.unsearch.dev
+          pytest backend/tests/smoke/ -v --base-url=https://staging-api.unsearch.dev
 
   # ========================================
   # Deploy to Production

.github/workflows/deploy-cf.yml

@@ -91,7 +91,7 @@ jobs:
       - uses: actions/setup-python@v5
         with: { python-version: "3.11" }
       - run: pip install httpx pytest
-      - run: pytest tests/e2e/test_prod_smoke.py -v
+      - run: pytest backend/tests/e2e/test_prod_smoke.py -v
         env:
           UNSEARCH_BASE_URL: https://api.unsearch.dev
           UNSEARCH_TEST_API_KEY: ${{ secrets.UNSEARCH_TEST_API_KEY }}

Makefile

@@ -56,20 +56,20 @@ setup:
 
 install:
 	@echo "📥 Installing dependencies..."
-	@pip install -r requirements.txt
+	@pip install -r backend/requirements.txt
 
 # Development
 dev:
 	@echo "🔧 Starting development server..."
-	@uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
+	@cd backend && uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
 
 worker:
 	@echo "👷 Starting Celery worker..."
-	@celery -A app.workers.tasks worker --loglevel=info
+	@cd backend && celery -A app.workers.tasks worker --loglevel=info
 
 flower:
 	@echo "🌸 Starting Flower monitoring..."
-	@celery -A app.workers.tasks flower
+	@cd backend && celery -A app.workers.tasks flower
 
 # Testing
 test:
@@ -95,27 +95,27 @@ test-coverage:
 # Code quality
 lint:
 	@echo "🔍 Running linting..."
-	@flake8 app/ tests/
-	@isort --check-only app/ tests/
-	@black --check app/ tests/
+	@flake8 backend/app backend/tests
+	@isort --check-only backend/app backend/tests
+	@black --check backend/app backend/tests
 
 format:
 	@echo "✨ Formatting code..."
-	@isort app/ tests/
-	@black app/ tests/
+	@isort backend/app backend/tests
+	@black backend/app backend/tests
 
 typecheck:
 	@echo "🔎 Running type checking..."
-	@mypy app/
+	@mypy backend/app
 
 # Database
 migrate:
 	@echo "🗄️ Running database migrations..."
-	@alembic upgrade head
+	@cd backend && alembic upgrade head
 
 migration:
 	@echo "📝 Creating new migration..."
-	@read -p "Migration message: " msg; alembic revision --autogenerate -m "$$msg"
+	@read -p "Migration message: " msg; cd backend && alembic revision --autogenerate -m "$$msg"
 
 # Docker
 docker-build:
@@ -202,7 +202,7 @@ dev-setup: env install migrate
 prod-check:
 	@echo "🔍 Running production readiness checks..."
 	@echo "Checking configuration..."
-	@python -c "from app.config import get_settings; s=get_settings(); print('✅ Configuration loaded')"
+	@cd backend && python -c "from app.config import get_settings; s=get_settings(); print('✅ Configuration loaded')"
 	@echo "Checking dependencies..."
 	@pip check
 	@echo "Running security audit..."

README.md

@@ -154,23 +154,29 @@ docker compose -f docker-compose.quickstart.yml up -d
 
 ```
 unsearch/
-├── app/                    # FastAPI backend — search, extract, verify, audit
-├── apps/
-│   ├── backend/            # FastAPI backend (monorepo layout — same code, packaged for Docker)
-│   ├── web/                # Next.js dashboard on Cloudflare Workers (@opennextjs/cloudflare)
-│   ├── sdk-ts/             # @unsearch/sdk — TypeScript SDK
-│   ├── sdk-py/             # unsearch — Python SDK (sync + async)
-│   ├── sdk-llamaindex/     # @unsearch/llamaindex — LlamaIndex retriever
-│   └── mcp-server/         # @unsearch/mcp-server — MCP server (P0 Week 3)
-├── workers/                # Cloudflare Workers edge — Hono router, MCP transport, Durable Objects, D1 schema
+├── backend/                # FastAPI backend — search, extract, verify, audit (Python 3.11+)
+│   ├── app/                #   Python module (`from app.X import Y`)
+│   ├── alembic/            #   Postgres migrations
+│   ├── tests/              #   pytest suite
+│   ├── Dockerfile          #   Self-host image
+│   └── Dockerfile.cloudflare #   CF Containers image
+├── apps/                   # TypeScript / Python SDK packages (pnpm workspace)
+│   ├── web/                #   Next.js dashboard on Cloudflare Workers (@opennextjs/cloudflare)
+│   ├── sdk-ts/             #   @unsearch/sdk — TypeScript SDK
+│   ├── sdk-py/             #   unsearch — Python SDK (sync + async)
+│   ├── sdk-llamaindex/     #   @unsearch/llamaindex — LlamaIndex retriever
+│   └── mcp-server/         #   @unsearch/mcp-server — MCP server (P0 Week 3)
+├── workers/                # Cloudflare Workers edge — Hono router, MCP transport, Durable Objects, D1 schema, containers.toml
+├── infra/                  # Operational config (self-host stack + CF Container sidecars)
+│   ├── nginx/              #   Reverse-proxy for self-host TLS
+│   ├── monitoring/         #   Prometheus + Grafana provisioning
+│   └── searxng/            #   SearXNG meta-search engine config (production settings.yml)
 ├── docs/                   # Architecture, API reference, strategy, ADRs, runbooks
-├── alembic/                # Postgres migrations (origin DB)
-├── searxng/                # SearXNG meta-search engine config
-├── monitoring/             # Prometheus + Grafana provisioning (self-host)
-└── docker-compose*.yml     # Self-host stacks
+├── scripts/                # Setup + ops scripts (manage.sh, setup-stripe.sh, …)
+└── docker-compose*.yml     # Self-host stacks (build context = root; mount paths from infra/)
 ```
 
-The architecture (Workers fronting FastAPI on Cloudflare Containers GA, with D1 / KV / Vectorize / R2 / Queues / Durable Objects + SearXNG sidecar) is documented in [`docs/cloudflare-architecture.md`](./docs/cloudflare-architecture.md). The five new ADRs that drove the 2026-05-28 reposition are at [`docs/adr/`](./docs/adr/README.md) (#0009 through #0013).
+The architecture (Workers fronting FastAPI on Cloudflare Containers GA, with D1 / KV / Vectorize / R2 / Queues / Durable Objects + SearXNG sidecar) is documented in [`docs/cloudflare-architecture.md`](./docs/cloudflare-architecture.md). The five new ADRs that drove the 2026-05-28 reposition are at [`docs/adr/`](./docs/adr/README.md) (#0009 through #0013). ADR-0006 has a 2026-05-28 amendment describing the directory restructure (`app/` → `backend/app/`, ops into `infra/`).
 
 ---
 
@@ -331,10 +337,10 @@ Stripe billing, SMTP, OAuth, monitoring — all documented in [`docs/configurati
 ## Development
 
 ```bash
-# Python backend
+# Python backend (everything backend-related lives in backend/)
 python -m venv venv && source venv/bin/activate
-pip install -r requirements.txt
-uvicorn app.main:app --reload --port 8000
+pip install -r backend/requirements.txt
+cd backend && uvicorn app.main:app --reload --port 8000
 
 # Frontend (Next.js on Workers)
 pnpm --filter @unsearch/web dev
@@ -348,9 +354,14 @@ cd apps/sdk-py && pip install -e ".[test]" && pytest -q
 # TypeScript SDK
 pnpm --filter @unsearch/sdk build && pnpm --filter @unsearch/sdk test
 
-# Backend tests
-pytest tests/unit/ -v --cov=app
-pytest tests/integration/ -v
+# Backend tests (from backend/ where pytest.ini lives)
+cd backend && pytest tests/unit/ -v --cov=app
+cd backend && pytest tests/integration/ -v
+
+# Or via the Makefile (handles the `cd backend` for you)
+make dev       # uvicorn with --reload
+make test      # full test suite
+make migrate   # alembic upgrade head
 ```
 
 Lint, type-check, and test commands are wired into CI ([.github/workflows/](./.github/workflows/)). Conventions live in [CLAUDE.md](./CLAUDE.md) and [CONTRIBUTING.md](./CONTRIBUTING.md).

Dockerfile → backend/Dockerfile

@@ -20,15 +20,16 @@ RUN useradd --create-home --shell /bin/bash app
 WORKDIR /app
 
 # Install Python dependencies
-COPY requirements.txt .
+# Build context = repo root; backend source lives under backend/.
+COPY backend/requirements.txt .
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r requirements.txt
 
 # Download NLTK data
 RUN python -c "import nltk; nltk.download('punkt'); nltk.download('stopwords')"
 
-# Copy application code
-COPY --chown=app:app app/ ./app/
+# Copy application code (host: backend/app/ → container: /app/app/)
+COPY --chown=app:app backend/app/ ./app/
 
 # Create necessary directories
 RUN mkdir -p /app/logs /app/data && \

Dockerfile.cloudflare → backend/Dockerfile.cloudflare

@@ -28,11 +28,12 @@ RUN useradd --create-home --shell /bin/bash --uid 10001 app
 WORKDIR /app
 
 # Python deps
-COPY requirements.txt .
+# Build context = repo root; backend source lives under backend/.
+COPY backend/requirements.txt .
 RUN pip install --upgrade pip && pip install -r requirements.txt
 
-# App source (omit tests, monitoring, alembic migrations from runtime image)
-COPY --chown=app:app app/ ./app/
+# App source (omit tests, alembic migrations from runtime image)
+COPY --chown=app:app backend/app/ ./app/
 COPY --chown=app:app scripts/ ./scripts/
 
 USER app

docker-compose.prod.yml

@@ -78,8 +78,8 @@ services:
 
   nginx:
     volumes:
-      - ./nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro
-      - ./nginx/ssl:/etc/nginx/ssl:ro
+      - ./infra/nginx/nginx.prod.conf:/etc/nginx/nginx.conf:ro
+      - ./infra/nginx/ssl:/etc/nginx/ssl:ro
       - /etc/letsencrypt:/etc/letsencrypt:ro
     logging:
       driver: "json-file"

docker-compose.quickstart.yml

@@ -11,7 +11,7 @@ services:
   api:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: backend/Dockerfile
     container_name: unsearch-api
     ports:
       - "8000:8000"

docker-compose.yml

@@ -1,6 +1,8 @@
 services:
   api:
-    build: .
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
     container_name: unsearch-api
     ports:
       - "8000:8000"
@@ -40,7 +42,7 @@ services:
       - SEARXNG_BASE_URL=http://searxng:8080/
       - SEARXNG_SECRET_KEY=${SEARXNG_SECRET_KEY:-changeme}
     volumes:
-      - ./searxng:/etc/searxng:rw
+      - ./infra/searxng:/etc/searxng:rw
     networks:
       - unsearch-net
     healthcheck:
@@ -96,7 +98,9 @@ services:
     restart: unless-stopped
 
   celery-worker:
-    build: .
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
     container_name: unsearch-celery-worker
     command: celery -A app.workers.tasks worker --loglevel=info --concurrency=4
     environment:
@@ -116,7 +120,9 @@ services:
     restart: unless-stopped
 
   celery-beat:
-    build: .
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
     container_name: unsearch-celery-beat
     command: celery -A app.workers.tasks beat --loglevel=info
     environment:
@@ -132,7 +138,9 @@ services:
     restart: unless-stopped
 
   flower:
-    build: .
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
     container_name: unsearch-flower
     command: celery -A app.workers.tasks flower --port=5555
     ports:
@@ -154,8 +162,8 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
-      - ./nginx/ssl:/etc/nginx/ssl:ro
+      - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./infra/nginx/ssl:/etc/nginx/ssl:ro
     depends_on:
       - api
     networks:

docs/adr/0001-cloudflare-native-edge-architecture.md

@@ -17,7 +17,7 @@ UnSearch v1 ran the classic "managed VPS + Postgres + Redis + Celery" stack. Thr
 Adopt **Cloudflare-native edge** as the default deploy target:
 
 - **Cloudflare Workers (Hono router)** front every request at `workers/src/index.ts`. Endpoints that don't need Python (KV cache hits, simple search proxying, auth checks, rate limiting) terminate at the edge.
-- **Cloudflare Containers** host the FastAPI origin (`Dockerfile.cloudflare`, `workers/containers.toml`) with auto-scale 0–10. Workloads that need Python's ecosystem (heavy scraping, complex orchestration, alembic migrations against the long-tail of features) proxy from the worker via a service binding.
+- **Cloudflare Containers** host the FastAPI origin (`backend/Dockerfile.cloudflare`, `workers/containers.toml`) with auto-scale 0–10. Workloads that need Python's ecosystem (heavy scraping, complex orchestration, alembic migrations against the long-tail of features) proxy from the worker via a service binding.
 - **D1** is the primary edge database (`workers/schema.sql`). Postgres remains as the local-dev origin DB and as an escape hatch for ops that need it.
 - **KV** for hot-path caches (auth, rate-limit counters, search cache hits).
 - **R2** for object storage (scraped HTML snapshots, large extracted artifacts).
@@ -34,7 +34,7 @@ The split between "lives at the edge" and "lives in the Container" is documented
 - **Pro:** Global p95 latency drops to <100ms for KV-cache hits, <200ms for Worker AI calls.
 - **Pro:** Free tier on the platform side maps neatly onto our 5,000-req/mo free tier on the product side.
 - **Con:** Self-host story is more complex than "docker compose up" — we keep `docker-compose.yml` working as the no-Cloudflare path (see [ADR-0005](./0005-apache-2-license-self-hostable-from-day-one.md)), but it deliberately leaves the edge-resident features (Vectorize, Workers AI) as optional.
-- **Con:** Cloudflare Containers is still maturing — direct bindings from inside Containers aren't available, so the FastAPI Container talks to D1 / KV / Queues over REST (see `app/services/core/d1_client.py`, `cache_kv.py`, `queue_producer.py`).
+- **Con:** Cloudflare Containers is still maturing — direct bindings from inside Containers aren't available, so the FastAPI Container talks to D1 / KV / Queues over REST (see `backend/app/services/core/d1_client.py`, `cache_kv.py`, `queue_producer.py`).
 - **Con:** Locks us into Cloudflare's specific quirks (Vectorize index size limits, Durable Object eviction semantics, Workers' 50ms CPU-time budget).
 
 ## Alternatives considered